Coverage for trlc/lexer.py: 100%

1#!/usr/bin/env python3

3# TRLC - Treat Requirements Like Code

6# This file is part of the TRLC Python Reference Implementation.

8# TRLC is free software: you can redistribute it and/or modify it

9# under the terms of the GNU General Public License as published by

10# the Free Software Foundation, either version 3 of the License, or

11# (at your option) any later version.

12#

13# TRLC is distributed in the hope that it will be useful, but WITHOUT

14# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY

15# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public

16# License for more details.

17#

18# You should have received a copy of the GNU General Public License

19# along with TRLC. If not, see <https://www.gnu.org/licenses/>.

21from fractions import Fraction

22from abc import ABCMeta, abstractmethod

24from trlc.errors import Location, Message_Handler

27def triple_quoted_string_value(raw_value):

28 # lobster-trace: LRM.Complex_String_Value

29 assert isinstance(raw_value, str)

30 assert len(raw_value) >= 6

31 assert raw_value.startswith("'''") or raw_value.startswith('"""')

32 assert raw_value[:3] == raw_value[-3:]

34 lines = raw_value[3:-3].strip().splitlines()

35 if not lines:

36 return ""

38 non_empty_lines = [line for line in lines if line.strip()]

40 value = lines[0]

41 common_ws = ""

42 common_len = 0

43 if len(non_empty_lines) >= 2:

44 # The loop below cannot complete by construction

45 for c in non_empty_lines[1]: # pragma: no cover

46 if c not in (" \t"):

47 break

48 common_ws += c

49 common_len += 1

50 else:

51 return value

53 for line in lines[2:]:

54 if not line.strip():

55 continue

56 for idx in range(common_len):

57 if idx < len(line) and line[idx] == common_ws[idx]:

58 pass

59 else:

60 common_len = idx

61 break

63 for line in lines[1:]:

64 value += "\n" + line[common_len:].rstrip()

66 return value

69class Source_Reference(Location):

70 def __init__(self, lexer, start_line, start_col, start_pos, end_pos):

71 assert isinstance(lexer, TRLC_Lexer)

72 assert isinstance(start_line, int)

73 assert isinstance(start_col, int)

74 assert isinstance(start_pos, int)

75 assert isinstance(end_pos, int)

76 assert 0 <= start_pos <= end_pos < lexer.length

77 super().__init__(lexer.file_name, start_line, start_col)

78 self.lexer = lexer

79 self.start_pos = start_pos

80 self.end_pos = end_pos

82 def text(self):

83 return self.lexer.content[self.start_pos : self.end_pos + 1]

85 def context_lines(self):

86 line = ""

87 n = self.start_pos

88 while n >= 0:

89 if self.lexer.content[n] == "\n":

90 break

91 line = self.lexer.content[n] + line

92 n -= 1

93 offset = self.start_pos - n - 1

94 n = self.start_pos + 1

95 while n < self.lexer.length:

96 if self.lexer.content[n] == "\n":

97 break

98 line = line + self.lexer.content[n]

99 n += 1

100 maxtrail = n - self.start_pos

101 tlen = self.end_pos + 1 - self.start_pos

102

103 stripped_line = line.lstrip()

104 stripped_offset = offset - (len(line) - len(stripped_line))

105

106 return [stripped_line, " " * stripped_offset + "^" * min(tlen, maxtrail)]

107

108 def get_end_location(self):

109 lines_in_between = self.lexer.content[self.start_pos : self.end_pos + 1].count(

110 "\n"

111 )

112 end_line = self.line_no + lines_in_between

113

114 end_col = self.end_pos + 1

115 for n in range(self.end_pos, 1, -1):

116 if self.lexer.content[n] == "\n":

117 end_col = max(self.end_pos - n, 1)

118 break

119

120 return Location(self.file_name, end_line, end_col)

121

122

123class Token_Base:

124 def __init__(self, location, kind, value):

125 assert isinstance(location, Location)

126 assert isinstance(kind, str)

127 self.location = location

128 self.kind = kind

129 self.value = value

130

131

132class Token(Token_Base):

133 KIND = {

134 "COMMENT": "comment",

135 "IDENTIFIER": "identifier",

136 "KEYWORD": "keyword",

137 "BRA": "opening parenthesis '('",

138 "KET": "closing parenthesis ')'",

139 "S_BRA": "opening bracket '['",

140 "S_KET": "closing bracket ']'",

141 "C_BRA": "opening brace '{'",

142 "C_KET": "closing brace '}'",

143 "COMMA": "comma ','",

144 "AT": "separtor '@'",

145 "SEMICOLON": "separator ';'",

146 "COLON": "separator ':'",

147 "DOT": ".",

148 "RANGE": "..",

149 "ASSIGN": "=",

150 "OPERATOR": "operator",

151 "ARROW": "->",

152 "INTEGER": "integer literal",

153 "DECIMAL": "decimal literal",

154 "STRING": "string literal",

155 }

156

157 def __init__(self, location, kind, value=None, ast_link=None):

158 assert kind in Token.KIND

159 if kind in ("COMMENT", "IDENTIFIER", "KEYWORD", "OPERATOR", "STRING"):

160 assert isinstance(value, str)

161 elif kind == "INTEGER":

162 assert isinstance(value, int)

163 elif kind == "DECIMAL":

164 assert isinstance(value, Fraction)

165 else:

166 assert value is None

167 super().__init__(location, kind, value)

168 self.ast_link = ast_link

169

170 def __repr__(self):

171 if self.value is None:

172 return "%s_Token" % self.kind

173 else:

174 return "%s_Token(%s)" % (self.kind, self.value)

175

176

177class Lexer_Base(metaclass=ABCMeta):

178 def __init__(self, mh, content):

179 assert isinstance(mh, Message_Handler)

180 assert isinstance(content, str)

181 self.mh = mh

182 self.content = content

183 self.length = len(self.content)

184 self.tokens = []

185

186 self.lexpos = -3

187 self.line_no = 0

188 self.col_no = 0

189 self.cc = None

190 self.nc = None

191 self.nnc = None

192

193 self.advance()

194 self.advance()

195

196 @staticmethod

197 def is_alpha(char):

198 # lobster-trace: LRM.Identifier

199 return char.isascii() and char.isalpha()

200

201 @staticmethod

202 def is_numeric(char):

203 # lobster-trace: LRM.Integers

204 # lobster-trace: LRM.Decimals

205 return char.isascii() and char.isdigit()

206

207 @staticmethod

208 def is_alnum(char):

209 # lobster-trace: LRM.Identifier

210 return char.isascii() and char.isalnum()

211

212 @abstractmethod

213 def file_location(self):

214 pass

215

216 @abstractmethod

217 def token(self):

218 pass

219

220 def skip_whitespace(self):

221 # lobster-trace: LRM.Whitespace

222 while self.nc and self.nc.isspace():

223 self.advance()

224 self.advance()

225

226 def advance(self):

227 self.lexpos += 1

228 if self.cc == "\n" or self.lexpos == 0:

229 self.line_no += 1

230 self.col_no = 0

231 if self.nc is not None:

232 self.col_no += 1

233 self.cc = self.nc

234 self.nc = self.nnc

235 self.nnc = (

236 self.content[self.lexpos + 2] if self.lexpos + 2 < self.length else None

237 )

238

239

240class TRLC_Lexer(Lexer_Base):

241 KEYWORDS = frozenset(

242 [

243 "abs",

244 "abstract",

245 "and",

246 "checks",

247 "else",

248 "elsif",

249 "enum",

250 "error",

251 "exists",

252 "extends",

253 "false",

254 "fatal",

255 "final",

256 "forall",

257 "freeze",

258 "if",

259 "implies",

260 "import",

261 "in",

262 "not",

263 "null",

264 "optional",

265 "or",

266 "package",

267 "section",

268 "separator",

269 "then",

270 "true",

271 "tuple",

272 "type",

273 "warning",

274 "xor",

275 ]

276 )

277

278 PUNCTUATION = {

279 "(": "BRA",

280 ")": "KET",

281 "{": "C_BRA",

282 "}": "C_KET",

283 "[": "S_BRA",

284 "]": "S_KET",

285 ",": "COMMA",

286 "@": "AT",

287 ":": "COLON",

288 ";": "SEMICOLON",

289 "/": "OPERATOR",

290 "%": "OPERATOR",

291 "+": "OPERATOR",

292 "-": "OPERATOR",

293 }

294

295 def __init__(self, mh, file_name, file_content=None):

296 assert isinstance(file_name, str)

297 assert isinstance(file_content, str) or file_content is None

298 self.file_name = file_name

299 if file_content is None:

300 # lobster-trace: LRM.File_Encoding

301 # lobster-trace: LRM.File_Encoding_Fixed

302 with open(file_name, "r", encoding="UTF-8") as fd:

303 try:

304 super().__init__(mh, fd.read())

305 except UnicodeDecodeError as err:

306 mh.lex_error(Location(file_name), str(err))

307 else:

308 super().__init__(mh, file_content)

309

310 def current_location(self):

311 # lobster-exclude: Utility function

312 return Source_Reference(

313 lexer=self,

314 start_line=self.line_no,

315 start_col=self.col_no,

316 start_pos=self.lexpos,

317 end_pos=self.lexpos,

318 )

319

320 def file_location(self):

321 # lobster-exclude: Utility function

322 return Location(self.file_name, 1, 1)

323

324 def token(self):

325 # Skip whitespace and move to the next char

326 self.skip_whitespace()

327

328 # Return if we're done

329 if self.cc is None:

330 return None

331

332 start_pos = self.lexpos

333 start_line = self.line_no

334 start_col = self.col_no

335

336 if self.cc == "/" and self.nc == "/":

337 # lobster-trace: LRM.Comments

338 kind = "COMMENT"

339 while self.cc and self.nc != "\n":

340 self.advance()

341

342 elif self.cc == "/" and self.nc == "*":

343 # lobster-trace: LRM.Comments

344 kind = "COMMENT"

345 while self.nc and not (self.cc == "*" and self.nc == "/"):

346 self.advance()

347 self.advance()

348

349 elif self.is_alpha(self.cc):

350 # lobster-trace: LRM.Identifier

351 kind = "IDENTIFIER"

352 while self.nc and (self.is_alnum(self.nc) or self.nc == "_"):

353 self.advance()

354

355 elif self.cc in TRLC_Lexer.PUNCTUATION:

356 # lobster-trace: LRM.Single_Delimiters

357 kind = TRLC_Lexer.PUNCTUATION[self.cc]

358

359 elif self.cc == "=":

360 # lobster-trace: LRM.Single_Delimiters

361 # lobster-trace: LRM.Double_Delimiters

362 # lobster-trace: LRM.Lexing_Disambiguation

363 if self.nc == ">":

364 kind = "ARROW"

365 self.advance()

366 elif self.nc == "=":

367 kind = "OPERATOR"

368 self.advance()

369 else:

370 kind = "ASSIGN"

371

372 elif self.cc == ".":

373 # lobster-trace: LRM.Single_Delimiters

374 # lobster-trace: LRM.Double_Delimiters

375 # lobster-trace: LRM.Lexing_Disambiguation

376 if self.nc == ".":

377 kind = "RANGE"

378 self.advance()

379 else:

380 kind = "DOT"

381

382 elif self.cc in ("<", ">"):

383 # lobster-trace: LRM.Single_Delimiters

384 # lobster-trace: LRM.Double_Delimiters

385 # lobster-trace: LRM.Lexing_Disambiguation

386 kind = "OPERATOR"

387 if self.nc == "=":

388 self.advance()

389

390 elif self.cc == "!":

391 # lobster-trace: LRM.Double_Delimiters

392 # lobster-trace: LRM.Lexing_Disambiguation

393 kind = "OPERATOR"

394 if self.nc == "=":

395 self.advance()

396 else:

397 self.mh.lex_error(self.current_location(), "malformed != operator")

398

399 elif self.cc == "*":

400 # lobster-trace: LRM.Single_Delimiters

401 # lobster-trace: LRM.Double_Delimiters

402 # lobster-trace: LRM.Lexing_Disambiguation

403 kind = "OPERATOR"

404 if self.nc == "*":

405 self.advance()

406

407 elif self.cc == '"':

408 # lobster-trace: LRM.Strings

409 kind = "STRING"

410 if self.nc == '"' and self.nnc == '"':

411 self.advance()

412 self.advance()

413 quotes_seen = 0

414 while quotes_seen < 3:

415 self.advance()

416 if self.cc == '"':

417 quotes_seen += 1

418 else:

419 quotes_seen = 0

420 if self.nc is None:

421 self.mh.lex_error(

422 Source_Reference(

423 lexer=self,

424 start_line=start_line,

425 start_col=start_col,

426 start_pos=start_pos,

427 end_pos=self.lexpos,

428 ),

429 "unterminated triple-quoted string",

430 )

431 else:

432 while self.nc != '"':

433 if self.nc is None:

434 self.mh.lex_error(

435 Source_Reference(

436 lexer=self,

437 start_line=start_line,

438 start_col=start_col,

439 start_pos=start_pos,

440 end_pos=self.lexpos,

441 ),

442 "unterminated string",

443 )

444 elif self.nc == "\n":

445 self.mh.lex_error(

446 Source_Reference(

447 lexer=self,

448 start_line=start_line,

449 start_col=start_col,

450 start_pos=start_pos,

451 end_pos=self.lexpos,

452 ),

453 "double quoted strings cannot include newlines",

454 )

455

456 self.advance()

457 if self.cc == "\\" and self.nc == '"':

458 self.advance()

459 self.advance()

460

461 elif self.cc == "'":

462 # lobster-trace: LRM.Strings

463 kind = "STRING"

464 for _ in range(2):

465 self.advance()

466 if self.cc != "'":

467 self.mh.lex_error(

468 Source_Reference(

469 lexer=self,

470 start_line=start_line,

471 start_col=start_col,

472 start_pos=start_pos,

473 end_pos=self.lexpos,

474 ),

475 "malformed triple-quoted string",

476 )

477 quotes_seen = 0

478 while quotes_seen < 3:

479 self.advance()

480 if self.cc == "'":

481 quotes_seen += 1

482 else:

483 quotes_seen = 0

484 if self.nc is None:

485 self.mh.lex_error(

486 Source_Reference(

487 lexer=self,

488 start_line=start_line,

489 start_col=start_col,

490 start_pos=start_pos,

491 end_pos=self.lexpos,

492 ),

493 "unterminated triple-quoted string",

494 )

495

496 elif self.is_numeric(self.cc):

497 # lobster-trace: LRM.Integers

498 # lobster-trace: LRM.Decimals

499 kind = "INTEGER"

500

501 if self.cc == "0" and self.nc == "b":

502 digits_allowed = "01"

503 digits_forbidden = "23456789abcdefABCDEF"

504 int_base = 2

505 require_digit = True

506 decimal_allowed = False

507 self.advance()

508 elif self.cc == "0" and self.nc == "x":

509 digits_allowed = "0123456789abcdefABCDEF"

510 digits_forbidden = ""

511 int_base = 16

512 require_digit = True

513 decimal_allowed = False

514 self.advance()

515 else:

516 digits_allowed = "0123456789"

517 digits_forbidden = "abcdefABCDEF"

518 int_base = 10

519 require_digit = False

520 decimal_allowed = True

521

522 while self.nc:

523 if self.nc in digits_allowed:

524 self.advance()

525 require_digit = False

526

527 elif self.nc in digits_forbidden:

528 self.mh.lex_error(

529 Source_Reference(

530 lexer=self,

531 start_line=start_line,

532 start_col=start_col,

533 start_pos=self.lexpos + 1,

534 end_pos=self.lexpos + 1,

535 ),

536 "%s is not a valid base %u digit" % (self.nc, int_base),

537 )

538

539 elif require_digit:

540 self.mh.lex_error(

541 Source_Reference(

542 lexer=self,

543 start_line=start_line,

544 start_col=start_col,

545 start_pos=self.lexpos + 1,

546 end_pos=self.lexpos + 1,

547 ),

548 "base %u digit is required here" % int_base,

549 )

550

551 elif self.nc == "_":

552 self.advance()

553 require_digit = True

554

555 elif self.nc == "." and self.nnc == ".":

556 # This is a range token, so that one can't be part

557 # of our number anymore

558 break

559

560 elif self.nc == ".":

561 self.advance()

562 if not decimal_allowed:

563 if int_base == 10:

564 msg = "decimal point is not allowed here"

565 else:

566 msg = (

567 "base %u integer may not contain a"

568 " decimal point" % int_base

569 )

570 self.mh.lex_error(

571 Source_Reference(

572 lexer=self,

573 start_line=start_line,

574 start_col=start_col,

575 start_pos=self.lexpos,

576 end_pos=self.lexpos,

577 ),

578 msg,

579 )

580 decimal_allowed = False

581 require_digit = True

582 kind = "DECIMAL"

583

584 else: # pragma: no cover

585 # This is actually a false

586 # alarm, this line is covered (it's the only

587 # normal way to exit this loop.

588 break

589

590 if require_digit:

591 self.mh.lex_error(

592 Source_Reference(

593 lexer=self,

594 start_line=start_line,

595 start_col=start_col,

596 start_pos=start_pos,

597 end_pos=self.lexpos,

598 ),

599 "unfinished base %u integer" % int_base,

600 )

601

602 else:

603 self.mh.lex_error(

604 self.current_location(), "unexpected character '%s'" % self.cc

605 )

606

607 sref = Source_Reference(

608 lexer=self,

609 start_line=start_line,

610 start_col=start_col,

611 start_pos=start_pos,

612 end_pos=min(self.lexpos, self.length - 1),

613 )

614

615 if kind == "IDENTIFIER":

616 value = sref.text()

617 if value in TRLC_Lexer.KEYWORDS:

618 # lobster-trace: LRM.TRLC_Keywords

619 kind = "KEYWORD"

620

621 elif kind == "OPERATOR":

622 value = sref.text()

623

624 elif kind == "STRING":

625 value = sref.text()

626 if value.startswith('"""'):

627 value = triple_quoted_string_value(value)

628 elif value.startswith('"'):

629 # lobster-trace: LRM.Simple_String_Value

630 value = value[1:-1]

631 value = value.replace('\\"', '"')

632 else:

633 value = triple_quoted_string_value(value)

634

635 elif kind == "INTEGER":

636 # lobster-trace: LRM.Integer_Values

637 base_text = sref.text().replace("_", "")

638 if int_base == 10:

639 value = int(base_text)

640 elif int_base == 2:

641 value = int(base_text[2:], base=2)

642 else:

643 value = int(base_text[2:], base=16)

644

645 elif kind == "DECIMAL":

646 # lobster-trace: LRM.Decimal_Values

647 value = Fraction(sref.text().replace("_", ""))

648

649 elif kind == "COMMENT":

650 value = sref.text()

651 if value.startswith("//"):

652 value = value[2:].strip()

653 else:

654 value = value[2:]

655 if value.endswith("*/"):

656 value = value[:-2]

657 value = value.strip()

658

659 else:

660 value = None

661

662 return Token(sref, kind, value)

663

664

665class Token_Stream(TRLC_Lexer):

666 def token(self):

667 tok = super().token()

668 if tok is not None:

669 self.tokens.append(tok)

670 return tok