Coverage for trlc/lexer

1#!/usr/bin/env python3

3# TRLC - Treat Requirements Like Code

6# This file is part of the TRLC Python Reference Implementation.

8# TRLC is free software: you can redistribute it and/or modify it

9# under the terms of the GNU General Public License as published by

10# the Free Software Foundation, either version 3 of the License, or

11# (at your option) any later version.

12#

13# TRLC is distributed in the hope that it will be useful, but WITHOUT

14# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY

15# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public

16# License for more details.

17#

18# You should have received a copy of the GNU General Public License

19# along with TRLC. If not, see <https://www.gnu.org/licenses/>.

21"""Lexer for Markdown TRLC (.trlc.md) files.

23Converts a Markdown representation of a TRLC requirements file into a

24token stream compatible with the TRLC parser.

26Markdown format

27---------------

28The file structure maps to TRLC constructs as follows:

30 # PackageName

31 → ``package PackageName``

33 import pkg

34 → ``import pkg``

36 ## Section name

37 → ``section "Section name" {``

39 <hr>

40 → Record separator (closes any open record within the section)

42 ### Record heading

43 → Starts a record; the heading text becomes the record identifier

44 (spaces and non-alphanumeric characters are replaced with ``_``).

46 Property table (under the ``###`` heading):

48 | Property | Value |

49 |----------|-------|

50 | type | Foo | ← ``type`` row gives the record type name

51 | field | value | ← subsequent rows are ``field = value`` assignments

53 #### Field heading

54 → Starts a free-text / string field whose name is the (lowercased)

55 heading identifier. All content that follows—including any Markdown

56 tables—is collected verbatim and emitted as a single STRING token.

57 The string field ends when the next heading (``##``/``###``/``####``),

58 ``<hr>``, or end-of-file is reached.

60Value inference

61---------------

62Values in the property table are interpreted as follows (in order):

64 * ``true`` / ``false`` / ``null`` → KEYWORD token

65 * Decimal integer (``42``) → INTEGER token

66 * Decimal number (``3.14``) → DECIMAL token

67 * Hex integer (``0x1A``) → INTEGER token

68 * Binary integer (``0b101``) → INTEGER token

69 * Dot-qualified identifier → chain of IDENTIFIER + DOT tokens

70 * Anything else → STRING token

71"""

73from fractions import Fraction

75from trlc.lexer import Token, TRLC_Lexer

76from trlc.errors import Location, Message_Handler

77from trlc.location_md import MD_Location

80class MD_Source_Reference(Location):

81 """A Location subclass that carries the source line and caret position

82 so Message_Handler can render the same visual caret output as TRLC."""

84 def __init__(self, file_name, line_no, col_no, source_line):

85 super().__init__(file_name, line_no, col_no)

86 self._source_line = source_line

88 def context_lines(self):

89 # Mirror Source_Reference.context_lines() from trlc/lexer.py:

90 # return [source_line_stripped, caret_string]

91 col = self.col_no if self.col_no else 1

92 stripped = self._source_line.lstrip()

93 leading = len(self._source_line) - len(stripped)

94 caret_col = max(col - 1 - leading, 0)

95 return [stripped, " " * caret_col + "^"]

98class MD_Lexer(TRLC_Lexer):

99 """Lexer that converts a ``.trlc.md`` file to a TRLC token stream.

100

101 The resulting token stream is compatible with :class:`trlc.parser.Parser`

102 when the parser is instantiated with a custom *lexer* argument.

103

104 Usage::

105

106 mh = Message_Handler()

107 lexer = MD_Lexer(mh, "path/to/file.trlc.md")

108 # pass lexer to Parser(mh, stab, file_name, ..., lexer=lexer)

109 """

110

111 # Boolean / null keywords (mirrors TRLC_Lexer.KEYWORDS)

112 KEYWORDS = frozenset(["true", "false", "null", "#", "##", "import"])

113

114 # Markdown-friendly aliases that map to TRLC block tokens.

115 MD_SECTION_START_TOKEN = "C_BRA"

116 MD_SECTION_END_TOKEN = "C_KET"

117

118 # ------------------------------------------------------------------ #

119 # Character classification (mirrors Lexer_Base / TRLC_Lexer helpers) #

120 # ------------------------------------------------------------------ #

121

122 @staticmethod

123 def _is_alpha(c):

124 return c.isascii() and c.isalpha()

125

126 @staticmethod

127 def _is_numeric(c):

128 return c.isascii() and c.isdigit()

129

130 @staticmethod

131 def _is_alnum(c):

132 return c.isascii() and c.isalnum()

133

134 @staticmethod

135 def _is_ident_start(c):

136 return c == "_" or (c.isascii() and c.isalpha())

137

138 @staticmethod

139 def _is_ident_cont(c):

140 return c == "_" or (c.isascii() and c.isalnum())

141

142 @staticmethod

143 def _is_hex_digit(c):

144 return c in "0123456789abcdefABCDEF"

145

146 # ------------------------------------------------------------------ #

147 # Value-token scanning helpers #

148 # ------------------------------------------------------------------ #

149

150 @staticmethod

151 def _scan_integer(text, start=0):

152 """Scan a run of decimal digits (and underscores) from *start*.

153

154 Returns the index one past the last scanned character, or ``-1``

155 when no digit is present at *start*.

156 """

157 i = start

158 n = len(text)

159 if i >= n or not text[i].isdigit():

160 return -1

161 while i < n and (text[i].isdigit() or text[i] == "_"):

162 i += 1

163 return i

164

165 @staticmethod

166 def _scan_hex(text, start=2):

167 """Scan hex digits from *start* (caller has consumed the ``0x`` prefix).

168

169 Returns the end index or ``-1`` if no valid hex digit at *start*.

170 """

171 i = start

172 n = len(text)

173 if i >= n or not MD_Lexer._is_hex_digit(text[i]):

174 return -1

175 while i < n and (MD_Lexer._is_hex_digit(text[i]) or text[i] == "_"):

176 i += 1

177 return i

178

179 @staticmethod

180 def _scan_binary(text, start=2):

181 """Scan binary digits from *start* (caller has consumed the ``0b`` prefix).

182

183 Returns the end index or ``-1`` if no valid binary digit at *start*.

184 """

185 i = start

186 n = len(text)

187 if i >= n or text[i] not in "01":

188 return -1

189 while i < n and (text[i] in "01" or text[i] == "_"):

190 i += 1

191 return i

192

193 @staticmethod

194 def _scan_ident(text, start=0):

195 """Scan one identifier segment from *text[start]*.

196

197 Returns the end index or ``-1`` when no identifier can start here.

198 """

199 i = start

200 n = len(text)

201 if i >= n or not MD_Lexer._is_ident_start(text[i]):

202 return -1

203 i += 1

204 while i < n and MD_Lexer._is_ident_cont(text[i]):

205 i += 1

206 return i

207

208 # ------------------------------------------------------------------ #

209 # Heading / structural helpers #

210 # ------------------------------------------------------------------ #

211

212 @staticmethod

213 def _parse_heading(line):

214 """Return ``(level, content)`` for a Markdown heading, or ``(0, None)``.

215

216 *level* is the number of leading ``#`` characters; *content* is the

217 stripped heading text. Levels 1–4 are meaningful to this lexer.

218 """

219 i = 0

220 while i < len(line) and line[i] == "#":

221 i += 1

222 if i == 0 or i >= len(line) or not line[i].isspace():

223 return 0, None

224 content = line[i:].strip()

225 if not content: 225 ↛ 226line 225 didn't jump to line 226 because the condition on line 225 was never true

226 return 0, None

227 return i, content

228

229 @staticmethod

230 def _is_hr(stripped):

231 """Return True if *stripped* is a markdown record separator line.

232

233 Accepts ``<hr>``, ``<hr/>``, `` ``, `` `` combinations used in

234 exported markdown such as ``<hr> <hr>``.

235 """

236 lower = stripped.lower().replace(" ", "")

237 if not lower:

238 return False

239

240 idx = 0

241 saw_hr = False

242 while idx < len(lower):

243 if lower.startswith("<hr>", idx):

244 idx += 4

245 saw_hr = True

246 elif lower.startswith("<hr/>", idx): 246 ↛ 247line 246 didn't jump to line 247 because the condition on line 246 was never true

247 idx += 5

248 saw_hr = True

249 elif lower.startswith(" ", idx):

250 idx += 4

251 elif lower.startswith(" ", idx): 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true

252 idx += 5

253 else:

254 return False

255

256 return saw_hr

257

258 # ------------------------------------------------------------------ #

259 # Construction #

260 # ------------------------------------------------------------------ #

261

262 def __init__(self, mh, file_name, file_content=None):

263 assert isinstance(mh, Message_Handler)

264 assert isinstance(file_name, str)

265

266 self.file_name = file_name

267

268 if file_content is None: 268 ↛ 272line 268 didn't jump to line 272 because the condition on line 268 was always true

269 with open(file_name, "r", encoding="UTF-8") as fd:

270 content = fd.read()

271 else:

272 assert isinstance(file_content, str)

273 content = file_content

274

275 # Initialise TRLC_Lexer to satisfy Parser lexer type checks.

276 super().__init__(mh, file_name, "")

277

278 self._md_tokens = [] # pre-computed token list

279 self._tok_index = 0

280

281 self._process(content)

282 # Keep parity with TRLC lexer API expected by linting/reporting.

283 self.tokens = self._md_tokens

284

285 # ------------------------------------------------------------------ #

286 # Lexer_Base interface #

287 # ------------------------------------------------------------------ #

288

289 def file_location(self):

290 return Location(self.file_name, 1, 1)

291

292 def token(self):

293 if self._tok_index < len(self._md_tokens):

294 tok = self._md_tokens[self._tok_index]

295 self._tok_index += 1

296 # print("MD_Lexer: end of token stream reached", tok)

297 return tok

298 return None

299

300 # ------------------------------------------------------------------ #

301 # Internal helpers #

302 # ------------------------------------------------------------------ #

303

304 def _loc(self, line_no, col_no=1):

305 return Location(self.file_name, line_no, col_no)

306

307 def _source_ref(self, line_no, col_no, source_line):

308 """Return a location that renders a caret line, like TRLC Source_Reference."""

309 return MD_Source_Reference(self.file_name, line_no, col_no, source_line)

310

311 def _emit(self, location, kind, value=None):

312 if kind == "STRING" and not hasattr(location, "text"):

313 location = MD_Location(

314 file_name=location.file_name,

315 line_no=location.line_no,

316 col_no=location.col_no,

317 token_text=value,

318 mh=self.mh,

319 )

320 self._md_tokens.append(Token(location, kind, value))

321

322 @staticmethod

323 def _heading_to_identifier(text):

324 """Convert arbitrary heading text to a valid TRLC identifier.

325

326 Replaces any run of non-alphanumeric characters with a single

327 underscore and strips leading/trailing underscores.

328 """

329 result = []

330 in_sep = False

331 for c in text.strip():

332 if c.isascii() and c.isalnum():

333 result.append(c)

334 in_sep = False

335 else:

336 if not in_sep:

337 result.append("_")

338 in_sep = True

339 return "".join(result).strip("_")

340

341 @staticmethod

342 def _is_valid_identifier(name):

343 """Return True when *name* matches TRLC identifier syntax."""

344 if not name:

345 return False

346 if not MD_Lexer._is_alpha(name[0]):

347 return False

348 return all(MD_Lexer._is_alnum(ch) or ch == "_" for ch in name[1:])

349

350 def _validate_identifier(self, name, loc_line, heading_prefix_len, source_line=""):

351 """Validate name against TRLC identifier rules.

352

353 Raises lex_error pointing at the first offending character,

354 with the same caret-line visual as TRLC's 'unexpected character X'.

355 """

356 if not name: 356 ↛ 357line 356 didn't jump to line 357 because the condition on line 356 was never true

357 self.mh.lex_error(

358 self._source_ref(loc_line, heading_prefix_len + 1, source_line),

359 "expected identifier",

360 )

361 if not MD_Lexer._is_alpha(name[0]): 361 ↛ 362line 361 didn't jump to line 362 because the condition on line 361 was never true

362 self.mh.lex_error(

363 self._source_ref(loc_line, heading_prefix_len + 1, source_line),

364 "unexpected character '%s'" % name[0],

365 )

366 for i, ch in enumerate(name[1:], 1):

367 if not (MD_Lexer._is_alnum(ch) or ch == "_"): 367 ↛ 368line 367 didn't jump to line 368 because the condition on line 367 was never true

368 self.mh.lex_error(

369 self._source_ref(loc_line, heading_prefix_len + 1 + i, source_line),

370 "unexpected character '%s'" % ch,

371 )

372

373 @staticmethod

374 def _is_separator_row(row):

375 """Return True if *row* is a Markdown table separator (``|---|``)."""

376 stripped = row.strip()

377 if not stripped.startswith("|"): 377 ↛ 378line 377 didn't jump to line 378 because the condition on line 377 was never true

378 return False

379 for c in stripped:

380 if c not in "|-: ":

381 return False

382 return True

383

384 @staticmethod

385 def _parse_table_row(row):

386 """Split a ``| key | value |`` row into (key, value) strings.

387

388 Returns ``None`` when the row cannot be parsed as a two-column

389 table entry.

390 """

391 stripped = row.strip()

392 if not stripped.startswith("|"): 392 ↛ 393line 392 didn't jump to line 393 because the condition on line 392 was never true

393 return None

394 # Split on "|", ignore the empty strings at both ends

395 parts = [p.strip() for p in stripped.split("|")]

396 # After split: ["", cell0, cell1, ..., ""]

397 cells = parts[1:-1]

398 if len(cells) >= 2: 398 ↛ 400line 398 didn't jump to line 400 because the condition on line 398 was always true

399 return cells[0], cells[1]

400 return None

401

402 def _emit_qualified_identifier(self, location, value):

403 """Emit IDENTIFIER[/DOT/IDENTIFIER...] tokens for a type name."""

404 parts = [p for p in value.split(".") if p]

405 for idx, part in enumerate(parts):

406 self._emit(location, "IDENTIFIER", part)

407 if idx < len(parts) - 1: 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true

408 self._emit(location, "DOT")

409

410 def _emit_value(self, raw_value, location):

411 """Emit one or more tokens representing a property value.

412

413 Inference rules are applied in the order documented in the

414 module docstring.

415 """

416 value = raw_value.strip()

417

418 # Boolean / null keywords

419 if value in MD_Lexer.KEYWORDS: 419 ↛ 420line 419 didn't jump to line 420 because the condition on line 419 was never true

420 self._emit(location, "KEYWORD", value)

421 return

422

423 # Positive integers

424 end = MD_Lexer._scan_integer(value)

425 if end == len(value) and end > 0: 425 ↛ 430line 425 didn't jump to line 430 because the condition on line 425 was always true

426 self._emit(location, "INTEGER", int(value.replace("_", "")))

427 return

428

429 # Positive decimals

430 dot_pos = value.find(".")

431 if dot_pos > 0:

432 int_end = MD_Lexer._scan_integer(value, 0)

433 if int_end == dot_pos:

434 dec_end = MD_Lexer._scan_integer(value, dot_pos + 1)

435 if dec_end == len(value) and dec_end > dot_pos + 1:

436 self._emit(location, "DECIMAL", Fraction(value.replace("_", "")))

437 return

438

439 # Hexadecimal 0x…

440 if value.startswith("0x") and len(value) > 2:

441 end = MD_Lexer._scan_hex(value)

442 if end == len(value):

443 self._emit(location, "INTEGER", int(value.replace("_", ""), 16))

444 return

445

446 # Binary 0b…

447 if value.startswith("0b") and len(value) > 2:

448 end = MD_Lexer._scan_binary(value)

449 if end == len(value):

450 self._emit(location, "INTEGER", int(value[2:].replace("_", ""), 2))

451 return

452

453 # Dot-qualified identifier or plain identifier

454 if value and MD_Lexer._is_ident_start(value[0]):

455 parts = []

456 i = 0

457 valid = True

458 while i <= len(value):

459 end = MD_Lexer._scan_ident(value, i)

460 if end < 0:

461 valid = False

462 break

463 parts.append(value[i:end])

464 i = end

465 if i == len(value):

466 break

467 if value[i] == ".":

468 i += 1

469 else:

470 valid = False

471 break

472 if valid and parts:

473 for idx, part in enumerate(parts):

474 self._emit(location, "IDENTIFIER", part)

475 if idx < len(parts) - 1:

476 self._emit(location, "DOT")

477 return

478

479 # Fall-back: treat the value as a plain string

480 self._emit(location, "STRING", value)

481

482 @staticmethod

483 def _looks_like_scalar_value(value):

484 """Return True when *value* should be inferred via _emit_value.

485

486 This is used for single-line ``####`` field bodies so short scalar

487 values (e.g. enum members, booleans, numbers) behave like table

488 properties, while free text remains a STRING.

489 """

490 text = value.strip()

491 if not text: 491 ↛ 492line 491 didn't jump to line 492 because the condition on line 491 was never true

492 return False

493

494 if text in ("true", "false", "null"): 494 ↛ 495line 494 didn't jump to line 495 because the condition on line 494 was never true

495 return True

496

497 # Integer / decimal

498 end = MD_Lexer._scan_integer(text)

499 if end == len(text) and end > 0: 499 ↛ 500line 499 didn't jump to line 500 because the condition on line 499 was never true

500 return True

501

502 dot_pos = text.find(".")

503 if dot_pos > 0: 503 ↛ 504line 503 didn't jump to line 504 because the condition on line 503 was never true

504 int_end = MD_Lexer._scan_integer(text, 0)

505 if int_end == dot_pos:

506 dec_end = MD_Lexer._scan_integer(text, dot_pos + 1)

507 if dec_end == len(text) and dec_end > dot_pos + 1:

508 return True

509

510 # Dot-qualified identifier (used e.g. for enum values)

511 if "." in text and MD_Lexer._is_ident_start(text[0]): 511 ↛ 512line 511 didn't jump to line 512 because the condition on line 511 was never true

512 i = 0

513 while i <= len(text):

514 end = MD_Lexer._scan_ident(text, i)

515 if end < 0:

516 return False

517 i = end

518 if i == len(text):

519 return True

520 if text[i] == ".":

521 i += 1

522 else:

523 return False

524

525 return False

526

527 # ------------------------------------------------------------------ #

528 # Main processing #

529 # ------------------------------------------------------------------ #

530

531 def _process(self, content):

532 """Transform *content* into the ``_md_tokens`` list."""

533

534 lines = content.splitlines()

535 total_lines = len(lines)

536

537 # ── Section tracking ─────────────────────────────────────────── #

538 in_section = False

539 imported_packages = []

540

541 # ── Record tracking ───────────────────────────────────────────── #

542 # When a ### heading is seen we buffer the name and then wait for

543 # the properties table to discover the record type.

544 in_record = False

545 pending_name = None # identifier string for the record

546 pending_name_loc = None # Location of the ### line

547 pending_props = [] # [(key, value, line_no)] before "type" row

548 record_type_found = False

549 props_first_row = False # True while we should skip the header row

550

551 # ── String-field tracking ─────────────────────────────────────── #

552 in_string_field = False

553 str_field_name = None

554 str_field_loc = None

555 str_field_lines = []

556 # #### fields seen before the "type" row are buffered here, then

557 # emitted inside the record after C_BRA (mirrors pending_props).

558 pending_string_fields = [] # [(name, loc, text, emit_as_scalar)]

559

560 # ── Helpers (closures) ────────────────────────────────────────── #

561

562 def flush_string_field():

563 nonlocal in_string_field, str_field_name, str_field_lines

564 if not in_string_field:

565 return

566 # Strip leading and trailing blank lines

567 while str_field_lines and not str_field_lines[0].strip(): 567 ↛ 568line 567 didn't jump to line 568 because the condition on line 567 was never true

568 str_field_lines.pop(0)

569 while str_field_lines and not str_field_lines[-1].strip():

570 str_field_lines.pop()

571 text = "\n".join(str_field_lines)

572

573 emit_as_scalar = ("\n" not in text and

574 MD_Lexer._looks_like_scalar_value(text))

575

576 if in_record: 576 ↛ 586line 576 didn't jump to line 586 because the condition on line 576 was always true

577 # Record already open – emit directly inside the block.

578 self._emit(str_field_loc, "IDENTIFIER", str_field_name)

579 self._emit(str_field_loc, "ASSIGN")

580 if emit_as_scalar: 580 ↛ 581line 580 didn't jump to line 581 because the condition on line 580 was never true

581 self._emit_value(text, str_field_loc)

582 else:

583 self._emit(str_field_loc, "STRING", text)

584 else:

585 # "type" row not yet seen – buffer until the record opens.

586 pending_string_fields.append(

587 (str_field_name, str_field_loc, text, emit_as_scalar))

588 in_string_field = False

589 str_field_name = None

590 str_field_lines = []

591

592 def flush_record(loc):

593 nonlocal in_record, pending_name, pending_props

594 nonlocal record_type_found, props_first_row

595 if not in_record:

596 # Nothing open – check for incomplete pending record

597 if pending_name is not None and not record_type_found: 597 ↛ 598line 597 didn't jump to line 598 because the condition on line 597 was never true

598 self.mh.error(

599 pending_name_loc,

600 "record heading '%s' has no 'type' property in its "

601 "property table; record will be skipped" % pending_name,

602 fatal=False,

603 )

604 pending_name = None

605 pending_props = []

606 pending_string_fields.clear()

607 record_type_found = False

608 props_first_row = False

609 return

610 flush_string_field()

611 self._emit(loc, "C_KET")

612 in_record = False

613 pending_name = None

614 pending_props = []

615 pending_string_fields.clear()

616 record_type_found = False

617 props_first_row = False

618

619 def open_section(name, loc):

620 nonlocal in_section

621 self._emit(loc, "KEYWORD", "##")

622 self._emit(loc, "STRING", name)

623 self._emit(loc, self.MD_SECTION_START_TOKEN)

624 in_section = True

625

626 def close_section(loc):

627 nonlocal in_section

628 if in_section:

629 self._emit(loc, self.MD_SECTION_END_TOKEN)

630 in_section = False

631

632 # ── Line-by-line scan ─────────────────────────────────────────── #

633

634 for i, line in enumerate(lines):

635 line_no = i + 1

636 loc = self._loc(line_no)

637 stripped = line.strip()

638

639 # ── While inside a string field, only headings / <hr> break out ─

640

641 if in_string_field:

642 level, _ = MD_Lexer._parse_heading(line)

643 if level in (2, 3, 4) or MD_Lexer._is_hr(stripped):

644 flush_string_field()

645 # fall through so the line is processed normally

646 elif not in_record and stripped.startswith("|"): 646 ↛ 651line 646 didn't jump to line 651 because the condition on line 646 was never true

647 # A property table row arrived while collecting a ####

648 # string field but before "type" is known. Close the

649 # string field (buffering its content) so the table row

650 # is processed normally below.

651 flush_string_field()

652 # fall through to table-row handling

653 else:

654 str_field_lines.append(line)

655 continue

656

657 # ── Heading dispatch (H1–H4) ─────────────────────────────────

658

659 _h_level, _h_content = MD_Lexer._parse_heading(line)

660

661 # ── H1: package declaration ──────────────────────────────────

662

663 if _h_level == 1:

664 parts = _h_content.split()

665 if len(parts) != 1: 665 ↛ 666line 665 didn't jump to line 666 because the condition on line 665 was never true

666 self.mh.lex_error(loc, "package heading must be '# <PackageName>'")

667 package_name = parts[0]

668 if ( 668 ↛ 674line 668 didn't jump to line 674 because the condition on line 668 was never true

669 not package_name or

670 not MD_Lexer._is_alpha(package_name[0]) or

671 any(not (MD_Lexer._is_alnum(ch) or ch == "_")

672 for ch in package_name[1:])

673 ):

674 self.mh.lex_error(loc, "invalid package name in markdown heading")

675 self._emit(loc, "KEYWORD", "#")

676 self._emit(loc, "IDENTIFIER", package_name)

677 continue

678

679 # ── H2: section ──────────────────────────────────────────────

680 # _parse_heading already distinguishes the levels by exact count

681 # of leading "#" characters, so no prefix-collision is possible.

682

683 if _h_level == 2:

684 flush_record(loc)

685 close_section(loc)

686 open_section(_h_content, loc)

687 continue

688

689 # ── H3: record heading ───────────────────────────────────────

690

691 if _h_level == 3:

692 flush_record(loc)

693 pending_name = _h_content.strip()

694 # heading_prefix_len = level(3) + 1 space

695 self._validate_identifier(pending_name, line_no, _h_level + 1, line)

696 pending_name_loc = loc

697 pending_props = []

698 record_type_found = False

699 props_first_row = True # skip the column-header row

700 continue

701

702 # ── H4: string field heading ─────────────────────────────────

703

704 if _h_level == 4:

705 # flush_string_field already called at the top of the loop

706 str_field_name = _h_content.strip()

707 # heading_prefix_len = level(4) + 1 space

708 self._validate_identifier(str_field_name, line_no, _h_level + 1, line)

709 str_field_loc = loc

710 str_field_lines = []

711 in_string_field = True

712 continue

713

714 # ── Horizontal rule ──────────────────────────────────────────

715

716 if MD_Lexer._is_hr(stripped):

717 flush_record(loc)

718 continue

719

720 # ── Table rows ───────────────────────────────────────────────

721

722 if stripped.startswith("|") and pending_name is not None:

723 # Skip the column-header row (first row after ###)

724 if props_first_row:

725 props_first_row = False

726 continue

727

728 # Skip Markdown separator rows (|---|---|)

729 if self._is_separator_row(stripped):

730 continue

731

732 row = self._parse_table_row(stripped)

733 if row is None: 733 ↛ 734line 733 didn't jump to line 734 because the condition on line 733 was never true

734 continue

735 key, value = row

736 if not key: 736 ↛ 737line 736 didn't jump to line 737 because the condition on line 736 was never true

737 continue

738

739 if key == "type":

740 # Emit: RecordType RecordName {

741 type_name = value.strip()

742 if "." not in type_name and len(imported_packages) == 1: 742 ↛ 743line 742 didn't jump to line 743 because the condition on line 742 was never true

743 type_name = imported_packages[0] + "." + type_name

744 self._emit_qualified_identifier(pending_name_loc, type_name)

745 self._emit(pending_name_loc, "IDENTIFIER", pending_name)

746 self._emit(pending_name_loc, "C_BRA")

747 in_record = True

748 record_type_found = True

749

750 # Flush any properties that arrived before "type"

751 for bkey, bval, bline in pending_props: 751 ↛ 752line 751 didn't jump to line 752 because the loop on line 751 never started

752 bloc = self._loc(bline)

753 self._emit(bloc, "IDENTIFIER", bkey)

754 self._emit(bloc, "ASSIGN")

755 self._emit_value(bval, bloc)

756 pending_props = []

757

758 # Flush any #### string fields that arrived before "type"

759 for fname, floc, ftext, fis_scalar in pending_string_fields: 759 ↛ 760line 759 didn't jump to line 760 because the loop on line 759 never started

760 self._emit(floc, "IDENTIFIER", fname)

761 self._emit(floc, "ASSIGN")

762 if fis_scalar:

763 self._emit_value(ftext, floc)

764 else:

765 self._emit(floc, "STRING", ftext)

766 pending_string_fields.clear()

767

768 elif record_type_found: 768 ↛ 775line 768 didn't jump to line 775 because the condition on line 768 was always true

769 self._emit(loc, "IDENTIFIER", key)

770 self._emit(loc, "ASSIGN")

771 self._emit_value(value, loc)

772

773 else:

774 # Buffer: "type" has not appeared yet

775 pending_props.append((key, value, line_no))

776

777 continue

778

779 # ── Import statement ─────────────────────────────────────────

780

781 if stripped.startswith("import "): 781 ↛ 782line 781 didn't jump to line 782 because the condition on line 781 was never true

782 parts = stripped.split()

783 if len(parts) == 2:

784 self._emit(loc, "KEYWORD", "import")

785 self._emit(loc, "IDENTIFIER", parts[1])

786 imported_packages.append(parts[1])

787 continue

788

789 # ── Everything else: delegate to TRLC_Lexer ──────────────────

790 # Tokenize the raw line so the parser receives real tokens and

791 # can report a meaningful error (e.g. "expected keyword #,

792 # encountered OPERATOR instead" for "* foo", or "encountered

793 # IDENTIFIER instead" for "sadsad foo").

794 if stripped: 794 ↛ 795line 794 didn't jump to line 795 because the condition on line 794 was never true

795 trlc_lex = TRLC_Lexer(self.mh, self.file_name, line)

796 tok = trlc_lex.token()

797 while tok is not None:

798 self._emit(

799 self._loc(line_no, tok.location.col_no),

800 tok.kind,

801 tok.value,

802 )

803 tok = trlc_lex.token()

804

805 # ── End-of-file cleanup ──────────────────────────────────────────

806

807 eof_loc = self._loc(total_lines + 1)

808 flush_record(eof_loc)

809 close_section(eof_loc)

Coverage for trlc/lexer_md.py: 58%

432 statements