Coverage for trlc/lexer_md.py: 89%
797 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-06 14:44 +0000
1#!/usr/bin/env python3
2#
3# TRLC - Treat Requirements Like Code
4# Copyright (C) 2026 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
5#
6# This file is part of the TRLC Python Reference Implementation.
7#
8# TRLC is free software: you can redistribute it and/or modify it
9# under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# TRLC is distributed in the hope that it will be useful, but WITHOUT
14# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
16# License for more details.
17#
18# You should have received a copy of the GNU General Public License
19# along with TRLC. If not, see <https://www.gnu.org/licenses/>.
21"""Lexer for Markdown TRLC (.trlc.md) files.
23Converts a Markdown representation of a TRLC requirements file into a
24token stream compatible with the TRLC parser.
26Markdown format
27---------------
28The file structure maps to TRLC constructs as follows:
30 # PackageName
31 → ``package PackageName``
33 import pkg
34 → ``import pkg``
36 The package heading and the import lines are the markdown spelling of
37 the TRLC preamble, so they follow the ``package_name`` and
38 ``import_clause`` grammar of the language: the name may be a nested
39 package name (``# ns.app``, ``import ns.base``) and an import may be a
40 wildcard (``import ns.*``).
42 ## Record heading
43 → Starts a record; the heading text becomes the record identifier
44 (spaces and non-alphanumeric characters are replaced with ``_``).
46 Property table (under the ``##`` heading):
48 | Property | Value |
49 |----------|-------|
50 | type | Foo | ← ``type`` row gives the record type name
51 | field | value | ← subsequent rows are ``field = value`` assignments
53 ### Field heading
54 → Starts a free-text / string field whose name is the (lowercased)
55 heading identifier. All content that follows—including any Markdown
56 tables—is collected verbatim and emitted as a single STRING token.
57 The string field ends when the next heading (``##``/``###``),
58 ``<hr>``, or end-of-file is reached.
60Value inference
61---------------
62Values in the property table are interpreted as follows (in order):
64 * ``true`` / ``false`` / ``null`` → KEYWORD token
65 * Decimal integer (``42``) → INTEGER token
66 * Decimal number (``3.14``) → DECIMAL token
67 * Hex integer (``0x1A``) → INTEGER token
68 * Binary integer (``0b101``) → INTEGER token
69 * Dot-qualified identifier → chain of IDENTIFIER + DOT tokens
70 * Anything else → STRING token
71"""
73import re
74from fractions import Fraction
76from trlc import ast as trlc_ast
77from trlc.errors import Location, Message_Handler
78from trlc.lexer import Token, TRLC_Lexer
79from trlc.location_md import MD_Location
82class MD_Source_Reference(Location):
83 """A Location subclass that carries the source line and caret position
84 so Message_Handler can render the same visual caret output as TRLC."""
86 def __init__(self, file_name, line_no, col_no, source_line):
87 super().__init__(file_name, line_no, col_no)
88 self._source_line = source_line
90 def context_lines(self):
91 # Mirror Source_Reference.context_lines() from trlc/lexer.py:
92 # return [source_line_stripped, caret_string]
93 col = self.col_no if self.col_no else 1
94 stripped = self._source_line.lstrip()
95 leading = len(self._source_line) - len(stripped)
96 caret_col = max(col - 1 - leading, 0)
97 return [stripped, " " * caret_col + "^"]
100class MD_Lexer(TRLC_Lexer):
101 """Lexer that converts a ``.trlc.md`` file to a TRLC token stream.
103 The resulting token stream is compatible with :class:`trlc.parser.Parser`
104 when the parser is instantiated with a custom *lexer* argument.
106 Usage::
108 mh = Message_Handler()
109 lexer = MD_Lexer(mh, "path/to/file.trlc.md")
110 # pass lexer to Parser(mh, stab, file_name, ..., lexer=lexer)
111 """
113 # Boolean / null keywords (mirrors TRLC_Lexer.KEYWORDS)
114 KEYWORDS = frozenset(["true", "false", "null", "#", "import"])
116 # Matches a genuine HTML line-break tag (``<br>``, ``<br/>``, ``<BR>``,
117 # ``<Br >``, ...). Used to recognize break tags as separators without
118 # also treating arbitrary ``<...>`` text the same way.
119 _BREAK_TAG_RE = re.compile(r"<[Bb][Rr]\s*/?>")
121 # Matches a ``<br>`` tag that sits directly between the ``import``
122 # keyword and the package name (e.g. ``import <br> base``). The ``<br>``
123 # separator feature is only valid *between or after* complete import
124 # clauses, never inside one, so this pattern is used to reject it.
125 _LEADING_BREAK_AFTER_IMPORT_RE = re.compile(r"\bimport\b\s*(<[Bb][Rr]\s*/?>)")
127 # ------------------------------------------------------------------ #
128 # Character classification (mirrors Lexer_Base / TRLC_Lexer helpers) #
129 # ------------------------------------------------------------------ #
131 @staticmethod
132 def _is_alpha(c):
133 return c.isascii() and c.isalpha()
135 @staticmethod
136 def _is_numeric(c):
137 return c.isascii() and c.isdigit()
139 @staticmethod
140 def _is_alnum(c):
141 return c.isascii() and c.isalnum()
143 @staticmethod
144 def _is_ident_start(c):
145 return c == "_" or (c.isascii() and c.isalpha())
147 @staticmethod
148 def _is_ident_cont(c):
149 return c == "_" or (c.isascii() and c.isalnum())
151 @staticmethod
152 def _is_hex_digit(c):
153 return c in "0123456789abcdefABCDEF"
155 # ------------------------------------------------------------------ #
156 # Value-token scanning helpers #
157 # ------------------------------------------------------------------ #
159 @staticmethod
160 def _scan_integer(text, start=0):
161 """Scan a run of decimal digits (and underscores) from *start*.
163 Returns the index one past the last scanned character, or ``-1``
164 when no digit is present at *start*.
165 """
166 i = start
167 n = len(text)
168 if i >= n or not text[i].isdigit():
169 return -1
170 while i < n and (text[i].isdigit() or text[i] == "_"):
171 i += 1
172 return i
174 @staticmethod
175 def _scan_hex(text, start=2):
176 """Scan hex digits from *start* (caller has consumed the ``0x`` prefix).
178 Returns the end index or ``-1`` if no valid hex digit at *start*.
179 """
180 i = start
181 n = len(text)
182 if i >= n or not MD_Lexer._is_hex_digit(text[i]): 182 ↛ 183line 182 didn't jump to line 183 because the condition on line 182 was never true
183 return -1
184 while i < n and (MD_Lexer._is_hex_digit(text[i]) or text[i] == "_"):
185 i += 1
186 return i
188 @staticmethod
189 def _scan_binary(text, start=2):
190 """Scan binary digits from *start* (caller has consumed the ``0b`` prefix).
192 Returns the end index or ``-1`` if no valid binary digit at *start*.
193 """
194 i = start
195 n = len(text)
196 if i >= n or text[i] not in "01": 196 ↛ 197line 196 didn't jump to line 197 because the condition on line 196 was never true
197 return -1
198 while i < n and (text[i] in "01" or text[i] == "_"):
199 i += 1
200 return i
202 @staticmethod
203 def _scan_ident(text, start=0):
204 """Scan one identifier segment from *text[start]*.
206 Returns the end index or ``-1`` when no identifier can start here.
207 """
208 i = start
209 n = len(text)
210 if i >= n or not MD_Lexer._is_ident_start(text[i]): 210 ↛ 211line 210 didn't jump to line 211 because the condition on line 210 was never true
211 return -1
212 i += 1
213 while i < n and MD_Lexer._is_ident_cont(text[i]):
214 i += 1
215 return i
217 # ------------------------------------------------------------------ #
218 # Heading / structural helpers #
219 # ------------------------------------------------------------------ #
221 @staticmethod
222 def _parse_heading(line):
223 """Return ``(level, content)`` for a Markdown heading, or ``(0, None)``.
225 *level* is the number of leading ``#`` characters; *content* is the
226 stripped heading text. Levels 1–4 are meaningful to this lexer.
227 """
228 i = 0
229 while i < len(line) and line[i] == "#":
230 i += 1
231 if i == 0 or i >= len(line) or not line[i].isspace():
232 return 0, None
233 content = line[i:].strip()
234 if not content: 234 ↛ 235line 234 didn't jump to line 235 because the condition on line 234 was never true
235 return 0, None
236 return i, content
238 @staticmethod
239 def _blank_out_break_tags(line):
240 """Replace real ``<br>`` tags with spaces of the same length.
242 This keeps column offsets stable while treating a break tag as
243 whitespace for tokenization, without silently accepting arbitrary
244 ``<...>`` text as a separator.
245 """
246 return MD_Lexer._BREAK_TAG_RE.sub(lambda m: " " * len(m.group(0)), line)
248 @staticmethod
249 def _is_hr(stripped):
250 """Return True if *stripped* is a markdown record separator line.
252 Accepts ``<hr>``, ``<hr/>``, ``<br>``, ``<br/>`` combinations used in
253 exported markdown such as ``<hr><br><hr>``.
254 """
255 lower = stripped.lower().replace(" ", "")
256 if not lower:
257 return False
259 idx = 0
260 saw_hr = False
261 while idx < len(lower):
262 if lower.startswith("<hr>", idx):
263 idx += 4
264 saw_hr = True
265 elif lower.startswith("<hr/>", idx):
266 idx += 5
267 saw_hr = True
268 elif lower.startswith("<br>", idx):
269 idx += 4
270 elif lower.startswith("<br/>", idx):
271 idx += 5
272 else:
273 return False
275 return saw_hr
277 # ------------------------------------------------------------------ #
278 # Construction #
279 # ------------------------------------------------------------------ #
281 def __init__(self, mh, file_name, file_content=None):
282 assert isinstance(mh, Message_Handler)
283 assert isinstance(file_name, str)
285 self.file_name = file_name
287 if file_content is None:
288 with open(file_name, "r", encoding="UTF-8") as fd:
289 content = fd.read()
290 else:
291 assert isinstance(file_content, str)
292 content = file_content
294 # Initialise TRLC_Lexer to satisfy Parser lexer type checks.
295 super().__init__(mh, file_name, "")
297 self._raw_content = content # kept for Phase 2 reprocessing
298 self._stab = None # set by prepare_phase2() after RSL
299 self._md_tokens = []
300 self._tok_index = 0
302 # Phase 1: emit only preamble tokens (# PackageName, import lines).
303 # Phase 2 is triggered by Source_Manager after parse_rsl_files().
304 self._process_preamble(content)
305 self._preamble_end_index = len(self._md_tokens)
306 self.tokens = self._md_tokens
308 # ------------------------------------------------------------------ #
309 # Lexer_Base interface #
310 # ------------------------------------------------------------------ #
312 def file_location(self):
313 return Location(self.file_name, 1, 1)
315 def token(self):
316 if self._tok_index < len(self._md_tokens):
317 tok = self._md_tokens[self._tok_index]
318 self._tok_index += 1
319 # print("MD_Lexer: end of token stream reached", tok)
320 return tok
321 return None
323 def _process_preamble(self, content):
324 """Phase 1: emit only preamble tokens (# PackageName, import lines).
326 Body processing is deferred to prepare_phase2() so that RSL types
327 are available when field values are tokenised.
328 """
329 lines = content.splitlines()
330 preamble = []
331 for line in lines:
332 stripped = line.strip()
333 if not stripped:
334 preamble.append(line)
335 continue
336 if stripped.startswith("# ") or stripped == "#":
337 preamble.append(line)
338 continue
339 if stripped.startswith("import "):
340 preamble.append(line)
341 continue
342 break # first non-preamble line stops Phase 1
343 self._process("\n".join(preamble))
345 def prepare_phase2(self, stab):
346 """Phase 2: reprocess the full content with RSL types available.
348 Called by Source_Manager after parse_rsl_files() so the stab is
349 fully populated. After this returns, the caller must re-prime the
350 parser token cursor (set ct=None and call advance() once).
351 """
352 self._stab = stab
353 self._md_tokens = []
354 self._tok_index = 0
355 self._process(self._raw_content) # full reprocessing with types
356 self._tok_index = self._preamble_end_index # skip preamble
357 self.tokens = self._md_tokens
359 def _resolve_record_type(self, type_name, package_name=None):
360 """Look up the Record_Type AST node in the stab for *type_name*."""
361 if self._stab is None or not type_name: 361 ↛ 362line 361 didn't jump to line 362 because the condition on line 361 was never true
362 return None
363 if "." in type_name:
364 # The package part may itself be dotted (nested packages), so
365 # only the last segment is the type name.
366 pkg_name, local_name = type_name.rsplit(".", 1)
367 elif package_name: 367 ↛ 370line 367 didn't jump to line 370 because the condition on line 367 was always true
368 pkg_name, local_name = package_name, type_name
369 else:
370 return None
371 pkg_simple = trlc_ast.Symbol_Table.simplified_name(pkg_name)
372 pkg = self._stab.table.get(pkg_simple)
373 if not isinstance(pkg, trlc_ast.Package):
374 return None
375 type_simple = trlc_ast.Symbol_Table.simplified_name(local_name)
376 record_type = pkg.symbols.table.get(type_simple)
377 if isinstance(record_type, trlc_ast.Record_Type):
378 return record_type
379 return None
381 @staticmethod
382 def _get_field_type(record_type_ast, field_name):
383 """Return the declared type of *field_name*, or None."""
384 if record_type_ast is None: 384 ↛ 385line 384 didn't jump to line 385 because the condition on line 384 was never true
385 return None
386 simple = trlc_ast.Symbol_Table.simplified_name(field_name)
387 component = record_type_ast.components.table.get(simple)
388 return component.n_typ if component is not None else None
390 @staticmethod
391 def _is_tuple_array_field(record_type_ast, field_name):
392 """Return True when *field_name* is declared as an array of tuples."""
393 typ = MD_Lexer._get_field_type(record_type_ast, field_name)
394 return isinstance(typ, trlc_ast.Array_Type) and isinstance(
395 typ.element_type, trlc_ast.Tuple_Type
396 )
398 @staticmethod
399 def _integer_tuple_array_arity(record_type_ast, field_name):
400 """Return the arity of a separator-form array of integer tuples."""
401 typ = MD_Lexer._get_field_type(record_type_ast, field_name)
402 if not isinstance(typ, trlc_ast.Array_Type) or not isinstance( 402 ↛ 405line 402 didn't jump to line 405 because the condition on line 402 was never true
403 typ.element_type, trlc_ast.Tuple_Type
404 ):
405 return None
407 tuple_type = typ.element_type
408 components = list(tuple_type.components.table.values())
409 if not tuple_type.has_separators() or not components:
410 return None
411 if not all(
412 isinstance(component.n_typ, trlc_ast.Builtin_Integer)
413 for component in components
414 ):
415 return None
416 return len(components)
418 @staticmethod
419 def _is_string_field(record_type_ast, field_name):
420 """Return True when *field_name* is declared as a plain String."""
421 typ = MD_Lexer._get_field_type(record_type_ast, field_name)
422 return typ is not None and isinstance(typ, trlc_ast.Builtin_String)
424 def _emit_field_value(
425 self, raw_value, location, record_type_ast=None, field_name=None
426 ):
427 """Type-aware field value emission (used in Phase 2).
429 Dispatch rules when RSL type info is available:
430 - String field → emit STRING directly (never try array/tuple)
431 - Tuple-array field → type-confirmed array tokenisation
432 - Any other field → existing _emit_value() heuristics
434 Falls back to _emit_value() heuristics when type is unknown.
435 """
436 if record_type_ast is not None:
437 if self._is_string_field(record_type_ast, field_name):
438 self._emit(location, "STRING", raw_value.strip())
439 return
440 if self._is_tuple_array_field(record_type_ast, field_name):
441 integer_tuple_arity = self._integer_tuple_array_arity(
442 record_type_ast, field_name
443 )
444 if self._check_bracket_array(raw_value, location, integer_tuple_arity):
445 return
446 if integer_tuple_arity is not None:
447 if self._maybe_emit_integer_tuple_array( 447 ↛ 451line 447 didn't jump to line 451 because the condition on line 447 was always true
448 raw_value, location, integer_tuple_arity
449 ):
450 return
451 self._report_invalid_array_entry(
452 raw_value,
453 location,
454 field_name,
455 integer_tuple_arity,
456 )
457 return
458 if not self._maybe_emit_array(
459 raw_value, location, allow_unqualified=True
460 ):
461 self._report_invalid_array_entry(raw_value, location, field_name)
462 return
463 self._emit_value(raw_value, location)
465 # ------------------------------------------------------------------ #
466 # Internal helpers #
467 # ------------------------------------------------------------------ #
469 def _loc(self, line_no, col_no=1):
470 return Location(self.file_name, line_no, col_no)
472 def _source_ref(self, line_no, col_no, source_line):
473 """Return a location that renders a caret line, like TRLC Source_Reference."""
474 return MD_Source_Reference(self.file_name, line_no, col_no, source_line)
476 def _emit(self, location, kind, value=None):
477 if kind == "STRING" and not hasattr(location, "text"):
478 location = MD_Location(
479 file_name=location.file_name,
480 line_no=location.line_no,
481 col_no=location.col_no,
482 token_text=value,
483 mh=self.mh,
484 )
485 self._md_tokens.append(Token(location, kind, value))
487 @staticmethod
488 def _heading_to_identifier(text):
489 """Convert arbitrary heading text to a valid TRLC identifier.
491 Replaces any run of non-alphanumeric characters with a single
492 underscore and strips leading/trailing underscores.
493 """
494 result = []
495 in_sep = False
496 for c in text.strip():
497 if c.isascii() and c.isalnum():
498 result.append(c)
499 in_sep = False
500 else:
501 if not in_sep:
502 result.append("_")
503 in_sep = True
504 return "".join(result).strip("_")
506 @staticmethod
507 def _is_valid_identifier(name):
508 """Return True when *name* matches TRLC identifier syntax."""
509 if not name:
510 return False
511 if not MD_Lexer._is_alpha(name[0]):
512 return False
513 return all(MD_Lexer._is_alnum(ch) or ch == "_" for ch in name[1:])
515 @staticmethod
516 def _skip_spaces(line, start):
517 """Return the 0-based offset of the first non-whitespace character
518 in *line* at or after *start*, so callers can locate a token
519 positionally instead of via a substring search that could match an
520 earlier, unrelated occurrence."""
521 rest = line[start:]
522 return start + (len(rest) - len(rest.lstrip()))
524 def _validate_identifier(self, name, loc_line, heading_prefix_len, source_line=""):
525 """Validate name against TRLC identifier rules.
527 Raises lex_error pointing at the first offending character,
528 with the same caret-line visual as TRLC's 'unexpected character X'.
529 """
530 if not name:
531 self.mh.lex_error(
532 self._source_ref(loc_line, heading_prefix_len + 1, source_line),
533 "expected identifier",
534 )
535 if not MD_Lexer._is_alpha(name[0]):
536 self.mh.lex_error(
537 self._source_ref(loc_line, heading_prefix_len + 1, source_line),
538 f"unexpected character '{name[0]}'",
539 )
540 for i, ch in enumerate(name[1:], 1):
541 if not (MD_Lexer._is_alnum(ch) or ch == "_"):
542 self.mh.lex_error(
543 self._source_ref(loc_line, heading_prefix_len + 1 + i, source_line),
544 f"unexpected character '{ch}'",
545 )
547 @staticmethod
548 def _is_separator_row(row):
549 """Return True if *row* is a Markdown table separator (``|---|``)."""
550 stripped = row.strip()
551 if not stripped.startswith("|"): 551 ↛ 552line 551 didn't jump to line 552 because the condition on line 551 was never true
552 return False
553 for c in stripped:
554 if c not in "|-: ":
555 return False
556 return True
558 @staticmethod
559 def _parse_table_row(row):
560 """Split a ``| key | value |`` row into (key, value) strings.
562 Returns ``None`` when the row cannot be parsed as a two-column
563 table entry.
564 """
565 stripped = row.strip()
566 if not stripped.startswith("|"):
567 return None
568 # Split on "|", ignore the empty strings at both ends
569 parts = [p.strip() for p in stripped.split("|")]
570 # After split: ["", cell0, cell1, ..., ""]
571 cells = parts[1:-1]
572 if len(cells) >= 2: 572 ↛ 574line 572 didn't jump to line 574 because the condition on line 572 was always true
573 return cells[0], cells[1]
574 return None
576 def _emit_qualified_identifier(self, location, value):
577 """Emit IDENTIFIER[/DOT/IDENTIFIER...] tokens for a type name."""
578 parts = [p for p in value.split(".") if p]
579 for idx, part in enumerate(parts):
580 self._emit(location, "IDENTIFIER", part)
581 if idx < len(parts) - 1:
582 self._emit(location, "DOT")
584 def _emit_package_name(
585 self, name, line_no, name_offset, source_line, allow_wildcard=False
586 ):
587 """Emit the token stream for a (possibly nested) package name.
589 Emits ``IDENTIFIER { DOT IDENTIFIER } [ DOT OPERATOR('*') ]``, i.e.
590 exactly what :meth:`trlc.parser.Parser.parse_dotted_name` and
591 :meth:`trlc.parser.Parser.parse_import_name` expect, so that
592 ``.trlc.md`` preambles share the TRLC grammar for package names.
594 *name_offset* is the 0-based column of *name* within *source_line*.
595 Each emitted token gets its own caret-capable location (rather than
596 one shared line-level location), so a parser error on e.g. the
597 second segment of ``ns.bad name`` points at that segment.
599 :returns: True if a trailing ``.*`` wildcard was emitted
600 """
601 # lobster-trace: LRM.Nested_Package_Names
602 # lobster-trace: LRM.Wildcard_Import
603 segments = name.split(".")
604 wildcard = allow_wildcard and len(segments) > 1 and segments[-1] == "*"
605 if wildcard:
606 segments = segments[:-1]
608 offset = name_offset
609 for idx, segment in enumerate(segments):
610 self._validate_identifier(segment, line_no, offset, source_line)
611 if idx:
612 self._emit(self._source_ref(line_no, offset, source_line), "DOT")
613 self._emit(
614 self._source_ref(line_no, offset + 1, source_line),
615 "IDENTIFIER",
616 segment,
617 )
618 offset += len(segment) + 1
620 if wildcard:
621 self._emit(self._source_ref(line_no, offset, source_line), "DOT")
622 self._emit(
623 self._source_ref(line_no, offset + 1, source_line), "OPERATOR", "*"
624 )
626 return wildcard
628 # Separator symbol token kinds (mirrors TRLC_Lexer.PUNCTUATION).
629 _SEPARATOR_PUNCTUATION = {"@": "AT", ":": "COLON", ";": "SEMICOLON"}
631 @staticmethod
632 def _strip_array_link_targets(raw_value):
633 """Keep link labels, skipping balanced and escaped destination parentheses."""
634 parts = []
635 cursor = 0
636 pattern = re.compile(r"\[([^\]]+)\]\(")
637 while cursor < len(raw_value):
638 match = pattern.search(raw_value, cursor)
639 if match is None:
640 break
641 start = match.start()
642 end = match.end()
643 depth = 1
644 while end < len(raw_value) and depth:
645 character = raw_value[end]
646 if character == "\\":
647 end += 2
648 continue
649 if character == "(":
650 depth += 1
651 elif character == ")":
652 depth -= 1
653 end += 1
654 if depth:
655 break
656 parts.append(raw_value[cursor:start])
657 parts.append(match.group(1))
658 cursor = end
659 parts.append(raw_value[cursor:])
660 return "".join(parts)
662 @staticmethod
663 def _normalize_array_value(raw_value):
664 """Normalize array value by converting various separators to comma format.
666 Handles:
667 - markdown links (``[text](url)`` -> ``text``)
668 - <br> tags (converted to newlines, then normalized)
669 - Multiple spaces around @, :, ; separators (normalized to single space)
670 - Identifier separators (multiple spaces collapsed to single space)
671 - Newlines (preserved initially for multiline detection, then normalized)
673 Returns normalized string with tuple refs as
674 ``"identifier <sep> integer, ..."``.
675 """
676 normalized = MD_Lexer._strip_array_link_targets(raw_value)
677 normalized = MD_Lexer._BREAK_TAG_RE.sub("\n", normalized)
679 # Split on newlines and commas, trim each part
680 parts = []
681 for line in normalized.split("\n"):
682 for item in line.split(","):
683 trimmed = item.strip()
684 if trimmed:
685 parts.append(trimmed)
687 # Normalize whitespace around each part:
688 # - punctuation separators (@, :, ;) get exactly one space each side
689 # - identifier separators: collapse multiple spaces to one
690 normalized_parts = []
691 for part in parts:
692 part = re.sub(r"\s*([@:;])\s*", r" \1 ", part)
693 part = re.sub(r" +", " ", part.strip())
694 normalized_parts.append(part)
696 return ", ".join(normalized_parts)
698 # One tuple-reference entry: identifier (optionally package-qualified)
699 # with an optional separator and version, e.g. ``Pkg.item @ 1``,
700 # ``item @ 1`` or ``item``. The separator is @, :, ; or a plain identifier.
701 _ARRAY_REF_RE = re.compile(
702 r"(?P<ref>[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*)"
703 r"(?:\s+(?P<sep>[@:;]|[a-zA-Z_]\w*)\s*(?P<ver>\d+))?"
704 )
706 # Chunk patterns for the multi-separator tuple scanner.
707 # A numeric chunk: hex, binary, or decimal (integer or float).
708 _TUPLE_NUM_RE = re.compile(
709 r"0[xX][0-9a-fA-F][0-9a-fA-F_]*"
710 r"|0[bB][01][01_]*"
711 r"|\d+(?:\.\d+)?"
712 )
713 # A separator chunk: @, :, ; or a word identifier.
714 _TUPLE_SEP_RE = re.compile(r"[@:;]|[a-zA-Z_]\w*")
716 @staticmethod
717 def _looks_like_array(raw_value, allow_unqualified=False):
718 """Check if value looks like tuple-reference array without emitting.
720 Expected format (separator may be @, :, ;, or any identifier)::
722 identifier[.identifier]* sep integer
723 identifier[.identifier]*
724 [, identifier[.identifier]* sep integer]*
726 Within Markdown requirement docs, a value may also appear wrapped in a
727 markdown link, e.g. ``[req@7](doc.md#req)`` or ``[req](doc.md#req)``.
728 The link text is parsed as the reference and the URL target is ignored.
730 When *allow_unqualified* is True, unqualified same-package references
731 (e.g. ``item @ 1`` or ``item``) are also accepted. Only pass True when
732 the field type is confirmed as a tuple-reference array.
733 """
734 normalized = MD_Lexer._normalize_array_value(raw_value)
735 parts = [p.strip() for p in normalized.split(",") if p.strip()]
736 if not parts: 736 ↛ 737line 736 didn't jump to line 737 because the condition on line 736 was never true
737 return False
738 return all(
739 MD_Lexer._matches_array_reference(part, allow_unqualified) for part in parts
740 )
742 @staticmethod
743 def _matches_array_reference(part, allow_unqualified=False):
744 """Return True when *part* is a reference, optionally versioned."""
745 return MD_Lexer._match_array_reference(part, allow_unqualified) is not None
747 @staticmethod
748 def _match_array_reference(part, allow_unqualified=False):
749 """Return the match for one array entry, or None when it is invalid.
751 Without *allow_unqualified* (type unknown) an entry must be
752 package-qualified and versioned, so that free text is never
753 mistaken for a reference.
754 """
755 match = MD_Lexer._ARRAY_REF_RE.fullmatch(part.strip())
756 if match is None:
757 return None
758 if not allow_unqualified and (
759 match.group("sep") is None or "." not in match.group("ref")
760 ):
761 return None
762 return match
764 @staticmethod
765 def _looks_like_simple_tuple(raw_value, expected_components=None):
766 """Check if value looks like a separator-form tuple: num [sep num]+.
768 Handles multi-separator patterns like 0x500:12345@6.1 or 1@2:3;4.
769 Returns True only when there is at least one separator (bare numbers
770 are handled by the scalar integer/decimal paths in _emit_value).
771 """
772 value = raw_value.strip()
773 pos = 0
774 n = len(value)
776 def skip_ws():
777 nonlocal pos
778 while pos < n and value[pos] in (" ", "\t"):
779 pos += 1
781 skip_ws()
782 m = MD_Lexer._TUPLE_NUM_RE.match(value, pos)
783 if not m:
784 return False
785 pos = m.end()
786 has_sep = False
787 component_count = 1
788 while pos < n:
789 skip_ws()
790 if pos >= n: 790 ↛ 791line 790 didn't jump to line 791 because the condition on line 790 was never true
791 break
792 m = MD_Lexer._TUPLE_SEP_RE.match(value, pos)
793 if not m: 793 ↛ 794line 793 didn't jump to line 794 because the condition on line 793 was never true
794 return False
795 pos = m.end()
796 has_sep = True
797 skip_ws()
798 m = MD_Lexer._TUPLE_NUM_RE.match(value, pos)
799 if not m: 799 ↛ 800line 799 didn't jump to line 800 because the condition on line 799 was never true
800 return False
801 pos = m.end()
802 component_count += 1
803 return (
804 pos == n
805 and has_sep
806 and (expected_components is None or component_count == expected_components)
807 )
809 def _maybe_emit_simple_tuple(self, raw_value, location):
810 """Try to emit tokens for a separator-form tuple: num [sep num]+.
812 Handles multi-separator patterns such as::
814 12345@42 → INTEGER AT INTEGER
815 0x500:12345@6.1 → INTEGER COLON INTEGER AT DECIMAL
816 1@2:3;4 → INTEGER AT INTEGER COLON INTEGER SEMICOLON INTEGER
818 Returns True if at least one separator was found and all tokens were
819 emitted, False to let _emit_value fall through to STRING.
820 """
821 value = raw_value.strip()
822 pos = 0
823 n = len(value)
824 chunks = [] # list of ('num', text) | ('sep', text)
826 def skip_ws():
827 nonlocal pos
828 while pos < n and value[pos] in (" ", "\t"):
829 pos += 1
831 skip_ws()
832 m = MD_Lexer._TUPLE_NUM_RE.match(value, pos)
833 if not m:
834 return False
835 chunks.append(("num", m.group()))
836 pos = m.end()
838 while pos < n:
839 skip_ws()
840 if pos >= n: 840 ↛ 841line 840 didn't jump to line 841 because the condition on line 840 was never true
841 break
842 m = MD_Lexer._TUPLE_SEP_RE.match(value, pos)
843 if not m: 843 ↛ 844line 843 didn't jump to line 844 because the condition on line 843 was never true
844 return False
845 chunks.append(("sep", m.group()))
846 pos = m.end()
847 skip_ws()
848 m = MD_Lexer._TUPLE_NUM_RE.match(value, pos)
849 if not m: 849 ↛ 850line 849 didn't jump to line 850 because the condition on line 849 was never true
850 return False
851 chunks.append(("num", m.group()))
852 pos = m.end()
854 if pos != n: 854 ↛ 855line 854 didn't jump to line 855 because the condition on line 854 was never true
855 return False
857 # Must have at least one separator so bare numbers fall through to
858 # the integer/decimal scalar checks in _emit_value.
859 has_sep = any(kind == "sep" for kind, _ in chunks)
860 if not has_sep: 860 ↛ 861line 860 didn't jump to line 861 because the condition on line 860 was never true
861 return False
863 for kind, text in chunks:
864 if kind == "num":
865 if "." in text:
866 self._emit(location, "DECIMAL", Fraction(text.replace("_", "")))
867 else:
868 self._emit(location, "INTEGER", self._parse_number(text))
869 else: # sep
870 sep_kind = MD_Lexer._SEPARATOR_PUNCTUATION.get(text, "IDENTIFIER")
871 sep_val = text if sep_kind == "IDENTIFIER" else None
872 self._emit(location, sep_kind, sep_val)
874 return True
876 def _maybe_emit_parentheses_tuple(self, raw_value, location):
877 """Try to emit tokens for a tuple in parentheses form: (val1, val2, val3).
879 Returns True if a tuple was emitted, False for fallback to STRING.
880 """
881 value = raw_value.strip()
882 if not (value.startswith("(") and value.endswith(")")):
883 return False
885 inner = value[1:-1].strip()
886 if not inner: 886 ↛ 888line 886 didn't jump to line 888 because the condition on line 886 was never true
887 # Empty tuple () - treat as error or skip
888 return False
890 # Split by commas and parse each value
891 self._emit(location, "BRA")
893 parts = [p.strip() for p in inner.split(",") if p.strip()]
894 for idx, part in enumerate(parts):
895 if idx > 0:
896 self._emit(location, "COMMA")
898 # Try to infer the value type for each part
899 self._emit_value(part, location)
901 self._emit(location, "KET")
902 return True
904 @staticmethod
905 def _parse_number(text):
906 """Parse a number string (decimal, hex, or binary).
908 Returns the integer value or None if parsing fails.
909 """
910 try:
911 text_clean = text.replace("_", "")
912 if text_clean.startswith("0x") or text_clean.startswith("0X"):
913 return int(text_clean, 16)
914 elif text_clean.startswith("0b") or text_clean.startswith("0B"): 914 ↛ 915line 914 didn't jump to line 915 because the condition on line 914 was never true
915 return int(text_clean, 2)
916 else:
917 return int(text_clean, 10)
918 except ValueError:
919 return None
921 def _check_bracket_array(self, raw_value, location, integer_tuple_arity=None):
922 """Detect bracket notation for tuple arrays and emit a clear error.
924 Bracket syntax like ``[Pkg.item @ 1, Pkg.item @ 2]`` is not supported
925 because brackets conflict with Markdown URL syntax.
927 Returns True when bracket array syntax was detected (error was emitted),
928 so the caller can skip further processing of this value.
929 """
930 value = raw_value.strip()
931 if not (value.startswith("[") and value.endswith("]")):
932 return False
933 inner = value[1:-1].strip()
934 # Only reject if the content inside the brackets actually looks like
935 # tuple references – avoids false positives on URLs containing '@'
936 if not MD_Lexer._looks_like_array(inner) and not ( 936 ↛ 940line 936 didn't jump to line 940 because the condition on line 936 was never true
937 integer_tuple_arity is not None
938 and MD_Lexer._looks_like_integer_tuple_array(inner, integer_tuple_arity)
939 ):
940 return False
941 self.mh.error(
942 location,
943 "bracket notation for tuple-reference arrays is not supported",
944 explanation=(
945 "Use comma-separated or newline-separated tuple references "
946 "without brackets instead, e.g. "
947 "'Pkg.item_a @ 1, Pkg.item_b @ 2'"
948 ),
949 fatal=False,
950 )
951 return True
953 @staticmethod
954 def _looks_like_integer_tuple_array(raw_value, expected_components):
955 normalized = MD_Lexer._normalize_array_value(raw_value)
956 parts = [part.strip() for part in normalized.split(",") if part.strip()]
957 return bool(parts) and all(
958 MD_Lexer._looks_like_simple_tuple(part, expected_components)
959 for part in parts
960 )
962 def _maybe_emit_integer_tuple_array(self, raw_value, location, expected_components):
963 normalized = MD_Lexer._normalize_array_value(raw_value)
964 parts = [part.strip() for part in normalized.split(",") if part.strip()]
965 if not parts or not all( 965 ↛ 969line 965 didn't jump to line 969 because the condition on line 965 was never true
966 MD_Lexer._looks_like_simple_tuple(part, expected_components)
967 for part in parts
968 ):
969 return False
971 self._emit(location, "S_BRA")
972 for idx, part in enumerate(parts):
973 if idx:
974 self._emit(location, "COMMA")
975 self._maybe_emit_simple_tuple(part, location)
976 self._emit(location, "S_KET")
977 return True
979 def _maybe_emit_array(self, raw_value, location, allow_unqualified=False):
980 """Try to emit array tokens for tuple-reference format.
982 When *allow_unqualified* is True, unqualified same-package references
983 (e.g. ``item @ 1`` or ``item``) are also accepted. Only safe when the
984 field type is confirmed as a tuple-reference array (Phase 2 path).
986 Recognises values of the form::
988 Package.item sep integer [, Package.item sep integer]*
989 item [sep integer]
991 Returns True if array was emitted, False for STRING fallback.
992 """
993 if not self._looks_like_array(raw_value, allow_unqualified=allow_unqualified):
994 return False
996 normalized = self._normalize_array_value(raw_value)
997 parts = [p.strip() for p in normalized.split(",")]
998 self._emit(location, "S_BRA")
1000 for idx, part in enumerate(parts):
1001 if idx > 0:
1002 self._emit(location, "COMMA")
1004 # _looks_like_array() has validated every entry already.
1005 match = MD_Lexer._match_array_reference(part, allow_unqualified)
1006 ident_parts = match.group("ref").split(".")
1007 for i, ident_part in enumerate(ident_parts):
1008 self._emit(location, "IDENTIFIER", ident_part)
1009 if i < len(ident_parts) - 1:
1010 self._emit(location, "DOT")
1012 sep = match.group("sep")
1013 if sep is not None:
1014 sep_kind = MD_Lexer._SEPARATOR_PUNCTUATION.get(sep, "IDENTIFIER")
1015 sep_val = sep if sep_kind == "IDENTIFIER" else None
1016 self._emit(location, sep_kind, sep_val)
1017 self._emit(location, "INTEGER", int(match.group("ver")))
1019 self._emit(location, "S_KET")
1020 return True
1022 def _report_invalid_array_entry(
1023 self, raw_value, location, field_name, integer_tuple_arity=None
1024 ):
1025 """Report the first entry of a tuple-array value that is not a reference.
1027 Emits an empty array afterwards so the parser does not add a
1028 misleading follow-up error for the same value.
1029 """
1030 normalized = self._normalize_array_value(raw_value)
1031 parts = [p.strip() for p in normalized.split(",") if p.strip()]
1032 if integer_tuple_arity is None: 1032 ↛ 1047line 1032 didn't jump to line 1047 because the condition on line 1032 was always true
1033 invalid = next(
1034 (p for p in parts if self._match_array_reference(p, True) is None),
1035 raw_value.strip(),
1036 )
1037 message = (
1038 f"invalid entry '{invalid}' in tuple-reference array field "
1039 f"'{field_name}'"
1040 )
1041 explanation = (
1042 "Use an identifier with an optional version, e.g. 'item' or "
1043 "'item @ 1'. Identifiers may only contain letters, digits "
1044 "and underscores."
1045 )
1046 else:
1047 invalid = next(
1048 (
1049 p
1050 for p in parts
1051 if not self._looks_like_simple_tuple(p, integer_tuple_arity)
1052 ),
1053 raw_value.strip(),
1054 )
1055 message = (
1056 f"invalid entry '{invalid}' in integer tuple array field '{field_name}'"
1057 )
1058 explanation = (
1059 "Use integer tuple values with the declared separators, "
1060 "e.g. '1234 @ 25'."
1061 )
1062 self.mh.error(
1063 location,
1064 message,
1065 explanation=explanation,
1066 fatal=False,
1067 )
1068 self._emit(location, "S_BRA")
1069 self._emit(location, "S_KET")
1071 def _emit_value(self, raw_value, location):
1072 """Emit one or more tokens representing a property value.
1074 Inference rules are applied in the order documented in the
1075 module docstring.
1076 """
1077 value = raw_value.strip()
1079 # Bracket array notation is not supported – emit a clear error
1080 if self._check_bracket_array(raw_value, location):
1081 return
1083 # Parentheses tuple form: (val1, val2, val3)
1084 if self._maybe_emit_parentheses_tuple(raw_value, location):
1085 return
1087 # Array of tuple references (before other checks)
1088 if self._maybe_emit_array(raw_value, location):
1089 return
1091 # Boolean / null keywords
1092 if value in MD_Lexer.KEYWORDS:
1093 self._emit(location, "KEYWORD", value)
1094 return
1096 # Positive integers
1097 end = MD_Lexer._scan_integer(value)
1098 if end == len(value) and end > 0:
1099 self._emit(location, "INTEGER", int(value.replace("_", "")))
1100 return
1102 # Positive decimals
1103 dot_pos = value.find(".")
1104 if dot_pos > 0:
1105 int_end = MD_Lexer._scan_integer(value, 0)
1106 if int_end == dot_pos:
1107 dec_end = MD_Lexer._scan_integer(value, dot_pos + 1)
1108 if dec_end == len(value) and dec_end > dot_pos + 1: 1108 ↛ 1113line 1108 didn't jump to line 1113 because the condition on line 1108 was always true
1109 self._emit(location, "DECIMAL", Fraction(value.replace("_", "")))
1110 return
1112 # Hexadecimal 0x…
1113 if value.startswith("0x") and len(value) > 2:
1114 end = MD_Lexer._scan_hex(value)
1115 if end == len(value):
1116 self._emit(location, "INTEGER", int(value.replace("_", ""), 16))
1117 return
1119 # Binary 0b…
1120 if value.startswith("0b") and len(value) > 2:
1121 end = MD_Lexer._scan_binary(value)
1122 if end == len(value): 1122 ↛ 1127line 1122 didn't jump to line 1127 because the condition on line 1122 was always true
1123 self._emit(location, "INTEGER", int(value[2:].replace("_", ""), 2))
1124 return
1126 # Simple tuple syntax (e.g., 12345@42, 0x500:6.1)
1127 if self._maybe_emit_simple_tuple(raw_value, location):
1128 return
1130 # Dot-qualified identifier or plain identifier
1131 if value and MD_Lexer._is_ident_start(value[0]): 1131 ↛ 1157line 1131 didn't jump to line 1157 because the condition on line 1131 was always true
1132 parts = []
1133 i = 0
1134 valid = True
1135 while i <= len(value): 1135 ↛ 1149line 1135 didn't jump to line 1149 because the condition on line 1135 was always true
1136 end = MD_Lexer._scan_ident(value, i)
1137 if end < 0: 1137 ↛ 1138line 1137 didn't jump to line 1138 because the condition on line 1137 was never true
1138 valid = False
1139 break
1140 parts.append(value[i:end])
1141 i = end
1142 if i == len(value):
1143 break
1144 if value[i] == ".":
1145 i += 1
1146 else:
1147 valid = False
1148 break
1149 if valid and parts:
1150 for idx, part in enumerate(parts):
1151 self._emit(location, "IDENTIFIER", part)
1152 if idx < len(parts) - 1:
1153 self._emit(location, "DOT")
1154 return
1156 # Fall-back: treat the value as a plain string
1157 self._emit(location, "STRING", value)
1159 @staticmethod
1160 def _looks_like_scalar_value(value):
1161 """Return True when *value* should be inferred via _emit_value.
1163 This is used for single-line ``###`` field bodies so short scalar
1164 values (e.g. enum members, booleans, numbers) behave like table
1165 properties, while free text remains a STRING.
1166 """
1167 text = value.strip()
1168 if not text: 1168 ↛ 1169line 1168 didn't jump to line 1169 because the condition on line 1168 was never true
1169 return False
1171 if text in ("true", "false", "null"): 1171 ↛ 1172line 1171 didn't jump to line 1172 because the condition on line 1171 was never true
1172 return True
1174 # Parentheses tuple form (simple detection)
1175 if text.startswith("(") and text.endswith(")"): 1175 ↛ 1176line 1175 didn't jump to line 1176 because the condition on line 1175 was never true
1176 return True
1178 # Integer / decimal
1179 end = MD_Lexer._scan_integer(text)
1180 if end == len(text) and end > 0: 1180 ↛ 1181line 1180 didn't jump to line 1181 because the condition on line 1180 was never true
1181 return True
1183 dot_pos = text.find(".")
1184 if dot_pos > 0:
1185 int_end = MD_Lexer._scan_integer(text, 0)
1186 if int_end == dot_pos: 1186 ↛ 1187line 1186 didn't jump to line 1187 because the condition on line 1186 was never true
1187 dec_end = MD_Lexer._scan_integer(text, dot_pos + 1)
1188 if dec_end == len(text) and dec_end > dot_pos + 1:
1189 return True
1191 # Simple tuple syntax (e.g., 12345@42)
1192 if MD_Lexer._looks_like_simple_tuple(text): 1192 ↛ 1193line 1192 didn't jump to line 1193 because the condition on line 1192 was never true
1193 return True
1195 # Dot-qualified identifier (used e.g. for enum values)
1196 if "." in text and MD_Lexer._is_ident_start(text[0]):
1197 i = 0
1198 while i <= len(text): 1198 ↛ 1210line 1198 didn't jump to line 1210 because the condition on line 1198 was always true
1199 end = MD_Lexer._scan_ident(text, i)
1200 if end < 0: 1200 ↛ 1201line 1200 didn't jump to line 1201 because the condition on line 1200 was never true
1201 return False
1202 i = end
1203 if i == len(text): 1203 ↛ 1204line 1203 didn't jump to line 1204 because the condition on line 1203 was never true
1204 return True
1205 if text[i] == ".":
1206 i += 1
1207 else:
1208 return False
1210 return False
1212 # ------------------------------------------------------------------ #
1213 # Main processing #
1214 # ------------------------------------------------------------------ #
1216 def _process(self, content):
1217 """Transform *content* into the ``_md_tokens`` list."""
1219 lines = content.splitlines()
1220 total_lines = len(lines)
1222 imported_packages = []
1224 # ── Type tracking for Phase 2 ─────────────────────────────────── #
1225 current_package_name = None # from # PackageName heading
1226 current_record_type_ast = None # resolved after type row found
1228 # ── Record tracking ───────────────────────────────────────────── #
1229 # When a ### heading is seen we buffer the name and then wait for
1230 # the properties table to discover the record type.
1231 in_record = False
1232 pending_name = None # identifier string for the record
1233 pending_name_loc = None # Location of the ### line
1234 pending_props = [] # [(key, value, line_no)] before "type" row
1235 record_type_found = False
1236 props_first_row = False # True while we should skip the header row
1238 # ── String-field tracking ─────────────────────────────────────── #
1239 in_string_field = False
1240 str_field_name = None
1241 str_field_loc = None
1242 str_field_lines = []
1243 str_field_first_content_line = None
1244 str_field_first_content_col = 1
1245 # ### fields seen before the "type" row are buffered here, then
1246 # emitted inside the record after C_BRA (mirrors pending_props).
1247 # Tuple: (name, loc, text, emit_as_scalar, is_array, is_bracket)
1248 pending_string_fields = []
1250 # ── Helpers (closures) ────────────────────────────────────────── #
1252 def flush_string_field():
1253 nonlocal in_string_field, str_field_name, str_field_lines
1254 nonlocal str_field_first_content_line, str_field_first_content_col
1255 if not in_string_field:
1256 return
1257 # Strip leading and trailing blank lines
1258 while str_field_lines and not str_field_lines[0].strip(): 1258 ↛ 1259line 1258 didn't jump to line 1259 because the condition on line 1258 was never true
1259 str_field_lines.pop(0)
1260 while str_field_lines and not str_field_lines[-1].strip():
1261 str_field_lines.pop()
1262 text = "\n".join(str_field_lines)
1264 emit_as_scalar = "\n" not in text and MD_Lexer._looks_like_scalar_value(
1265 text
1266 )
1268 value_loc = str_field_loc
1269 if str_field_first_content_line is not None: 1269 ↛ 1275line 1269 didn't jump to line 1275 because the condition on line 1269 was always true
1270 value_loc = self._loc(
1271 str_field_first_content_line,
1272 str_field_first_content_col,
1273 )
1275 if in_record: 1275 ↛ 1285line 1275 didn't jump to line 1285 because the condition on line 1275 was always true
1276 # Record already open – emit directly inside the block.
1277 self._emit(str_field_loc, "IDENTIFIER", str_field_name)
1278 self._emit(str_field_loc, "ASSIGN")
1279 self._emit_field_value(
1280 text, value_loc, current_record_type_ast, str_field_name
1281 )
1282 else:
1283 # "type" row not yet seen – buffer until the record opens.
1284 # Check if it looks like an array (but don't emit yet)
1285 is_bracket = (
1286 not emit_as_scalar
1287 and text.strip().startswith("[")
1288 and text.strip().endswith("]")
1289 and "@" in text
1290 )
1291 is_array = (
1292 not emit_as_scalar
1293 and not is_bracket
1294 and MD_Lexer._looks_like_array(text)
1295 )
1296 pending_string_fields.append(
1297 (
1298 str_field_name,
1299 value_loc,
1300 text,
1301 emit_as_scalar,
1302 is_array,
1303 is_bracket,
1304 )
1305 )
1306 in_string_field = False
1307 str_field_name = None
1308 str_field_lines = []
1309 str_field_first_content_line = None
1310 str_field_first_content_col = 1
1312 def flush_record(loc):
1313 nonlocal in_record, pending_name, pending_props
1314 nonlocal record_type_found, props_first_row
1315 if not in_record:
1316 # Nothing open – check for incomplete pending record
1317 if pending_name is not None and not record_type_found:
1318 self.mh.error(
1319 pending_name_loc,
1320 f"record heading '{pending_name}' has no 'type' property"
1321 " in its property table; record will be skipped",
1322 fatal=False,
1323 )
1324 pending_name = None
1325 pending_props = []
1326 pending_string_fields.clear()
1327 record_type_found = False
1328 props_first_row = False
1329 return
1330 flush_string_field()
1331 self._emit(loc, "C_KET")
1332 in_record = False
1333 pending_name = None
1334 pending_props = []
1335 pending_string_fields.clear()
1336 record_type_found = False
1337 props_first_row = False
1339 # ── Line-by-line scan ─────────────────────────────────────────── #
1341 for i, line in enumerate(lines):
1342 line_no = i + 1
1343 loc = self._loc(line_no)
1344 stripped = line.strip()
1346 # ── While inside a string field, only headings / <hr> break out ─
1348 if in_string_field:
1349 level, _ = MD_Lexer._parse_heading(line)
1350 if level in (2, 3) or MD_Lexer._is_hr(stripped):
1351 flush_string_field()
1352 # fall through so the line is processed normally
1353 elif not in_record and stripped.startswith("|"): 1353 ↛ 1358line 1353 didn't jump to line 1358 because the condition on line 1353 was never true
1354 # A property table row arrived while collecting a ###
1355 # string field but before "type" is known. Close the
1356 # string field (buffering its content) so the table row
1357 # is processed normally below.
1358 flush_string_field()
1359 # fall through to table-row handling
1360 else:
1361 if str_field_first_content_line is None and stripped:
1362 line_stripped = line.lstrip()
1363 str_field_first_content_line = line_no
1364 str_field_first_content_col = len(line) - len(line_stripped) + 1
1365 str_field_lines.append(line)
1366 continue
1368 # ── Heading dispatch (H1–H3) ─────────────────────────────────
1370 _h_level, _h_content = MD_Lexer._parse_heading(line)
1372 # ── H1: package declaration ──────────────────────────────────
1374 if _h_level == 1:
1375 parts = _h_content.split()
1376 if len(parts) != 1:
1377 self.mh.lex_error(loc, "package heading must be '# <PackageName>'")
1378 package_name = parts[0]
1379 self._emit(loc, "KEYWORD", "#")
1380 # Package name starts right after the '#' and its following
1381 # whitespace; computed positionally so a name that happens
1382 # to reoccur earlier on the line (e.g. inside "##") can't
1383 # be found at the wrong offset.
1384 name_offset = MD_Lexer._skip_spaces(line, _h_level)
1385 self._emit_package_name(package_name, line_no, name_offset, line)
1386 current_package_name = package_name
1387 continue
1389 # ── H2: record heading ────────────────────────────────────────
1390 # _parse_heading already distinguishes the levels by exact count
1391 # of leading "#" characters, so no prefix-collision is possible.
1393 if _h_level == 2:
1394 flush_record(loc)
1395 pending_name = _h_content.strip()
1396 self._validate_identifier(pending_name, line_no, _h_level + 1, line)
1397 pending_name_loc = loc
1398 pending_props = []
1399 record_type_found = False
1400 props_first_row = True # skip the column-header row
1401 continue
1403 # ── H3: string field heading ─────────────────────────────────
1405 if _h_level == 3:
1406 # flush_string_field already called at the top of the loop
1407 str_field_name = _h_content.strip()
1408 self._validate_identifier(str_field_name, line_no, _h_level + 1, line)
1409 str_field_loc = loc
1410 str_field_lines = []
1411 str_field_first_content_line = None
1412 str_field_first_content_col = 1
1413 in_string_field = True
1414 continue
1416 if _h_level >= 4:
1417 self.mh.lex_error(
1418 loc,
1419 "heading level 4 and above is not valid in .trlc.md; use "
1420 "'##' for records and '###' for fields",
1421 )
1422 continue
1424 # ── Horizontal rule ──────────────────────────────────────────
1426 if MD_Lexer._is_hr(stripped):
1427 flush_record(loc)
1428 continue
1430 # ── Table rows ───────────────────────────────────────────────
1432 if stripped.startswith("|") and pending_name is not None:
1433 # Skip the column-header row (first row after ###)
1434 if props_first_row:
1435 props_first_row = False
1436 continue
1438 # Skip Markdown separator rows (|---|---|)
1439 if self._is_separator_row(stripped):
1440 continue
1442 row = self._parse_table_row(stripped)
1443 if row is None: 1443 ↛ 1444line 1443 didn't jump to line 1444 because the condition on line 1443 was never true
1444 continue
1445 key, value = row
1446 if not key: 1446 ↛ 1447line 1446 didn't jump to line 1447 because the condition on line 1446 was never true
1447 continue
1449 if key == "type":
1450 # Emit: RecordType RecordName {
1451 type_name = value.strip()
1452 if "." not in type_name and len(imported_packages) == 1:
1453 type_name = imported_packages[0] + "." + type_name
1454 self._emit_qualified_identifier(loc, type_name)
1455 self._emit(pending_name_loc, "IDENTIFIER", pending_name)
1456 self._emit(pending_name_loc, "C_BRA")
1457 in_record = True
1458 record_type_found = True
1459 current_record_type_ast = self._resolve_record_type(
1460 type_name, package_name=current_package_name
1461 )
1463 # Flush any properties that arrived before "type"
1464 for bkey, bval, bline in pending_props: 1464 ↛ 1465line 1464 didn't jump to line 1465 because the loop on line 1464 never started
1465 bloc = self._loc(bline)
1466 self._emit(bloc, "IDENTIFIER", bkey)
1467 self._emit(bloc, "ASSIGN")
1468 self._emit_field_value(
1469 bval, bloc, current_record_type_ast, bkey
1470 )
1471 pending_props = []
1473 # Flush any ### string fields that arrived
1474 # before "type"
1475 for ( 1475 ↛ 1483line 1475 didn't jump to line 1483 because the loop on line 1475 never started
1476 fname,
1477 floc,
1478 ftext,
1479 _fis_scalar,
1480 _fis_array,
1481 _fis_bracket,
1482 ) in pending_string_fields:
1483 self._emit(floc, "IDENTIFIER", fname)
1484 self._emit(floc, "ASSIGN")
1485 self._emit_field_value(
1486 ftext, floc, current_record_type_ast, fname
1487 )
1488 pending_string_fields.clear()
1490 elif record_type_found:
1491 self._emit(loc, "IDENTIFIER", key)
1492 self._emit(loc, "ASSIGN")
1493 self._emit_field_value(value, loc, current_record_type_ast, key)
1495 else:
1496 # Buffer: "type" has not appeared yet
1497 pending_props.append((key, value, line_no))
1499 continue
1501 # ── Import statement ─────────────────────────────────────────
1503 if stripped.startswith("import "):
1504 # A <br> tag is only a valid separator *between* or *after*
1505 # complete "import <name>" clauses; reject one wedged
1506 # between the "import" keyword and its package name (e.g.
1507 # "import <br> base") instead of silently discarding it.
1508 bad_tag = MD_Lexer._LEADING_BREAK_AFTER_IMPORT_RE.search(line)
1509 if bad_tag is not None:
1510 self.mh.lex_error(
1511 self._source_ref(line_no, bad_tag.start(1) + 1, line),
1512 "a <br> tag is not allowed between 'import' and the"
1513 " package name",
1514 )
1516 # Replace real HTML break tags with spaces so adjacent imports
1517 # still parse as separate clauses without shifting column
1518 # offsets; leave arbitrary "<...>" text alone.
1519 scrubbed = MD_Lexer._blank_out_break_tags(line)
1520 for match in re.finditer(r"import\s+\S+", scrubbed):
1521 import_text = match.group(0)
1522 parts = import_text.split()
1523 if len(parts) != 2: 1523 ↛ 1524line 1523 didn't jump to line 1524 because the condition on line 1523 was never true
1524 continue
1526 kw_loc = self._loc(line_no, match.start() + 1)
1527 self._emit(kw_loc, "KEYWORD", "import")
1528 # Compute each package offset from its own match so extra
1529 # whitespace after "import" does not skew locations.
1530 name_offset = MD_Lexer._skip_spaces(
1531 scrubbed, match.start() + len("import")
1532 )
1533 wildcard = self._emit_package_name(
1534 parts[1],
1535 line_no,
1536 name_offset,
1537 line,
1538 allow_wildcard=True,
1539 )
1540 # Only a plain import identifies a single package that
1541 # unqualified type names can be completed with.
1542 if not wildcard:
1543 imported_packages.append(parts[1])
1544 continue
1546 # ── Everything else: delegate to TRLC_Lexer ──────────────────
1547 # Tokenize the raw line so the parser receives real tokens and
1548 # can report a meaningful error (e.g. "expected keyword #,
1549 # encountered OPERATOR instead" for "* foo", or "encountered
1550 # IDENTIFIER instead" for "sadsad foo").
1551 if stripped: 1551 ↛ 1552line 1551 didn't jump to line 1552 because the condition on line 1551 was never true
1552 trlc_lex = TRLC_Lexer(self.mh, self.file_name, line)
1553 tok = trlc_lex.token()
1554 while tok is not None:
1555 self._emit(
1556 self._loc(line_no, tok.location.col_no),
1557 tok.kind,
1558 tok.value,
1559 )
1560 tok = trlc_lex.token()
1562 # ── End-of-file cleanup ──────────────────────────────────────────
1564 eof_loc = self._loc(total_lines + 1)
1565 flush_record(eof_loc)