Coverage for lobster/config/lexer.py: 12%

1#!/usr/bin/env python3 

2# 

3# LOBSTER - Lightweight Open BMW Software Traceability Evidence Report 

4# Copyright (C) 2022-2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) 

5# 

6# This program is free software: you can redistribute it and/or modify 

7# it under the terms of the GNU Affero General Public License as 

8# published by the Free Software Foundation, either version 3 of the 

9# License, or (at your option) any later version. 

10# 

11# This program is distributed in the hope that it will be useful, but 

12# WITHOUT ANY WARRANTY; without even the implied warranty of 

13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 

14# Affero General Public License for more details. 

15# 

16# You should have received a copy of the GNU Affero General Public 

17# License along with this program. If not, see 

18# <https://www.gnu.org/licenses/>. 

19 

20import sys 

21import os.path 

22 

23from lobster import errors 

24from lobster import location 

25 

26 

27class Token: 

28 def __init__(self, kind, text, loc): 

29 self.kind = kind 

30 self.text = text 

31 self.loc = loc 

32 

33 def value(self): 

34 if self.kind == "STRING": 

35 return self.text[1:-1] 

36 else: 

37 return self.text 

38 

39 def __repr__(self): 

40 return "Token(%s, %s, %s)" % (self.kind, 

41 self.text, 

42 self.loc) 

43 

44 

45class Lexer: 

46 def __init__(self, mh, file_name): 

47 assert isinstance(mh, errors.Message_Handler) 

48 assert isinstance(file_name, str) 

49 assert os.path.isfile(file_name) 

50 

51 self.file_name = file_name 

52 self.mh = mh 

53 

54 with open(file_name, "r", encoding="UTF-8") as fd: 

55 self.content = fd.read() 

56 self.length = len(self.content) 

57 

58 self.lexpos = -1 

59 self.line_nr = 1 

60 self.cc = None 

61 self.nc = self.content[0] if self.length > 0 else None 

62 

63 def advance(self): 

64 self.lexpos += 1 

65 if self.cc == "\n": 

66 self.line_nr += 1 

67 self.cc = self.nc 

68 if self.lexpos + 1 < self.length: 

69 self.nc = self.content[self.lexpos + 1] 

70 else: 

71 self.nc = None 

72 

73 def error(self, message): 

74 loc = location.File_Reference(filename = self.file_name, 

75 line = self.line_nr) 

76 self.mh.lex_error(loc, message) 

77 

78 def token(self): 

79 # Skip comments and whitespace 

80 while True: 

81 while self.nc and self.nc.isspace(): 

82 self.advance() 

83 if self.nc is None: 

84 return None 

85 self.advance() 

86 

87 if self.cc == "#": 

88 while self.cc and self.cc != "\n": 

89 self.advance() 

90 else: 

91 break 

92 

93 kind = None 

94 t_start = self.lexpos 

95 

96 if self.cc == "{": 

97 kind = "C_BRA" 

98 elif self.cc == "}": 

99 kind = "C_KET" 

100 elif self.cc == ":": 

101 kind = "COLON" 

102 elif self.cc == ",": 

103 kind = "COMMA" 

104 elif self.cc == ";": 

105 kind = "SEMI" 

106 elif self.cc == '"': 

107 kind = "STRING" 

108 self.advance() 

109 while self.cc != '"': 

110 self.advance() 

111 if self.cc in (None, "\n"): 

112 self.error("unterminated string") 

113 elif self.cc.isalpha(): 

114 kind = "KEYWORD" 

115 while self.nc.isalpha() or self.nc == "_": 

116 self.advance() 

117 else: 

118 self.error("unexpected character: '%s'" % self.cc) 

119 

120 t_end = self.lexpos 

121 

122 return Token( 

123 kind = kind, 

124 text = self.content[t_start : t_end + 1], 

125 loc = location.File_Reference(filename = self.file_name, 

126 line = self.line_nr)) 

127 

128 

129def sanity_test(): 

130 mh = errors.Message_Handler() 

131 lexer = Lexer(mh, sys.argv[1]) 

132 try: 

133 while True: 

134 tok = lexer.token() 

135 if tok is None: 

136 break 

137 print(tok) 

138 except errors.LOBSTER_Error: 

139 return 1 

140 return 0 

141 

142 

143if __name__ == "__main__": 

144 sanity_test()