|
2 | 2 | #define RBS__LEXER_H |
3 | 3 |
|
4 | 4 | enum TokenType { |
5 | | - NullType, /* (Nothing) */ |
6 | | - pEOF, /* EOF */ |
7 | | - ErrorToken, /* Error */ |
8 | | - |
9 | | - pLPAREN, /* ( */ |
10 | | - pRPAREN, /* ) */ |
11 | | - pCOLON, /* : */ |
12 | | - pCOLON2, /* :: */ |
13 | | - pLBRACKET, /* [ */ |
14 | | - pRBRACKET, /* ] */ |
15 | | - pLBRACE, /* { */ |
16 | | - pRBRACE, /* } */ |
17 | | - pHAT, /* ^ */ |
18 | | - pARROW, /* -> */ |
19 | | - pFATARROW, /* => */ |
20 | | - pCOMMA, /* , */ |
21 | | - pBAR, /* | */ |
22 | | - pAMP, /* & */ |
23 | | - pSTAR, /* * */ |
24 | | - pSTAR2, /* ** */ |
25 | | - pDOT, /* . */ |
26 | | - pDOT3, /* ... */ |
27 | | - pBANG, /* ! */ |
28 | | - pQUESTION, /* ? */ |
29 | | - pLT, /* < */ |
30 | | - pEQ, /* = */ |
31 | | - |
32 | | - kALIAS, /* alias */ |
33 | | - kATTRACCESSOR, /* attr_accessor */ |
34 | | - kATTRREADER, /* attr_reader */ |
35 | | - kATTRWRITER, /* attr_writer */ |
36 | | - kBOOL, /* bool */ |
37 | | - kBOT, /* bot */ |
38 | | - kCLASS, /* class */ |
39 | | - kDEF, /* def */ |
40 | | - kEND, /* end */ |
41 | | - kEXTEND, /* extend */ |
42 | | - kFALSE, /* false */ |
43 | | - kIN, /* in */ |
44 | | - kINCLUDE, /* include */ |
45 | | - kINSTANCE, /* instance */ |
46 | | - kINTERFACE, /* interface */ |
47 | | - kMODULE, /* module */ |
48 | | - kNIL, /* nil */ |
49 | | - kOUT, /* out */ |
50 | | - kPREPEND, /* prepend */ |
51 | | - kPRIVATE, /* private */ |
52 | | - kPUBLIC, /* public */ |
53 | | - kSELF, /* self */ |
54 | | - kSINGLETON, /* singleton */ |
55 | | - kTOP, /* top */ |
56 | | - kTRUE, /* true */ |
57 | | - kTYPE, /* type */ |
58 | | - kUNCHECKED, /* unchecked */ |
59 | | - kUNTYPED, /* untyped */ |
60 | | - kVOID, /* void */ |
61 | | - kUSE, /* use */ |
62 | | - kAS, /* as */ |
63 | | - k__TODO__, /* __todo__ */ |
64 | | - kATRBS, /* @rbs */ |
65 | | - kSKIP, /* skip */ |
66 | | - kRETURN, /* return */ |
67 | | - |
68 | | - tLIDENT, /* Identifiers starting with lower case */ |
69 | | - tUIDENT, /* Identifiers starting with upper case */ |
70 | | - tULIDENT, /* Identifiers starting with `_` followed by upper case */ |
71 | | - tULLIDENT, /* Identifiers starting with `_` followed by lower case */ |
72 | | - tGIDENT, /* Identifiers starting with `$` */ |
73 | | - tAIDENT, /* Identifiers starting with `@` */ |
74 | | - tA2IDENT, /* Identifiers starting with `@@` */ |
75 | | - tBANGIDENT, /* Identifiers ending with `!` */ |
76 | | - tEQIDENT, /* Identifiers ending with `=` */ |
77 | | - tQIDENT, /* Quoted identifier */ |
78 | | - pAREF_OPR, /* [] */ |
79 | | - tOPERATOR, /* Operator identifier */ |
80 | | - |
81 | | - tCOMMENT, /* Comment */ |
82 | | - tLINECOMMENT, /* Comment of all line */ |
83 | | - tINLINECOMMENT, /* Comment in inline decl starting with -- */ |
84 | | - |
85 | | - tTRIVIA, /* Trivia tokens -- space and new line */ |
86 | | - |
87 | | - tDQSTRING, /* Double quoted string */ |
88 | | - tSQSTRING, /* Single quoted string */ |
89 | | - tINTEGER, /* Integer */ |
90 | | - tSYMBOL, /* Symbol */ |
91 | | - tDQSYMBOL, /* Double quoted symbol */ |
92 | | - tSQSYMBOL, /* Single quoted symbol */ |
93 | | - tANNOTATION, /* Annotation */ |
| 5 | + NullType, /* (Nothing) */ |
| 6 | + pEOF, /* EOF */ |
| 7 | + ErrorToken, /* Error */ |
| 8 | + |
| 9 | + pLPAREN, /* ( */ |
| 10 | + pRPAREN, /* ) */ |
| 11 | + pCOLON, /* : */ |
| 12 | + pCOLON2, /* :: */ |
| 13 | + pLBRACKET, /* [ */ |
| 14 | + pRBRACKET, /* ] */ |
| 15 | + pLBRACE, /* { */ |
| 16 | + pRBRACE, /* } */ |
| 17 | + pHAT, /* ^ */ |
| 18 | + pARROW, /* -> */ |
| 19 | + pFATARROW, /* => */ |
| 20 | + pCOMMA, /* , */ |
| 21 | + pBAR, /* | */ |
| 22 | + pAMP, /* & */ |
| 23 | + pSTAR, /* * */ |
| 24 | + pSTAR2, /* ** */ |
| 25 | + pDOT, /* . */ |
| 26 | + pDOT3, /* ... */ |
| 27 | + pBANG, /* ! */ |
| 28 | + pQUESTION, /* ? */ |
| 29 | + pLT, /* < */ |
| 30 | + pEQ, /* = */ |
| 31 | + |
| 32 | + kALIAS, /* alias */ |
| 33 | + kATTRACCESSOR, /* attr_accessor */ |
| 34 | + kATTRREADER, /* attr_reader */ |
| 35 | + kATTRWRITER, /* attr_writer */ |
| 36 | + kBOOL, /* bool */ |
| 37 | + kBOT, /* bot */ |
| 38 | + kCLASS, /* class */ |
| 39 | + kDEF, /* def */ |
| 40 | + kEND, /* end */ |
| 41 | + kEXTEND, /* extend */ |
| 42 | + kFALSE, /* false */ |
| 43 | + kIN, /* in */ |
| 44 | + kINCLUDE, /* include */ |
| 45 | + kINSTANCE, /* instance */ |
| 46 | + kINTERFACE, /* interface */ |
| 47 | + kMODULE, /* module */ |
| 48 | + kNIL, /* nil */ |
| 49 | + kOUT, /* out */ |
| 50 | + kPREPEND, /* prepend */ |
| 51 | + kPRIVATE, /* private */ |
| 52 | + kPUBLIC, /* public */ |
| 53 | + kSELF, /* self */ |
| 54 | + kSINGLETON, /* singleton */ |
| 55 | + kTOP, /* top */ |
| 56 | + kTRUE, /* true */ |
| 57 | + kTYPE, /* type */ |
| 58 | + kUNCHECKED, /* unchecked */ |
| 59 | + kUNTYPED, /* untyped */ |
| 60 | + kVOID, /* void */ |
| 61 | + kUSE, /* use */ |
| 62 | + kAS, /* as */ |
| 63 | + k__TODO__, /* __todo__ */ |
| 64 | + kATRBS, /* @rbs */ |
| 65 | + kSKIP, /* skip */ |
| 66 | + kRETURN, /* return */ |
| 67 | + |
| 68 | + tLIDENT, /* Identifiers starting with lower case */ |
| 69 | + tUIDENT, /* Identifiers starting with upper case */ |
| 70 | + tULIDENT, /* Identifiers starting with `_` followed by upper case */ |
| 71 | + tULLIDENT, /* Identifiers starting with `_` followed by lower case */ |
| 72 | + tGIDENT, /* Identifiers starting with `$` */ |
| 73 | + tAIDENT, /* Identifiers starting with `@` */ |
| 74 | + tA2IDENT, /* Identifiers starting with `@@` */ |
| 75 | + tBANGIDENT, /* Identifiers ending with `!` */ |
| 76 | + tEQIDENT, /* Identifiers ending with `=` */ |
| 77 | + tQIDENT, /* Quoted identifier */ |
| 78 | + pAREF_OPR, /* [] */ |
| 79 | + tOPERATOR, /* Operator identifier */ |
| 80 | + |
| 81 | + tCOMMENT, /* Comment */ |
| 82 | + tLINECOMMENT, /* Comment of all line */ |
| 83 | + tINLINECOMMENT, /* Comment in inline decl starting with -- */ |
| 84 | + |
| 85 | + tTRIVIA, /* Trivia tokens -- space and new line */ |
| 86 | + |
| 87 | + tDQSTRING, /* Double quoted string */ |
| 88 | + tSQSTRING, /* Single quoted string */ |
| 89 | + tINTEGER, /* Integer */ |
| 90 | + tSYMBOL, /* Symbol */ |
| 91 | + tDQSYMBOL, /* Double quoted symbol */ |
| 92 | + tSQSYMBOL, /* Single quoted symbol */ |
| 93 | + tANNOTATION, /* Annotation */ |
94 | 94 | }; |
95 | 95 |
|
96 | 96 | /** |
@@ -128,12 +128,12 @@ typedef struct { |
128 | 128 | * */ |
129 | 129 | typedef struct { |
130 | 130 | VALUE string; |
131 | | - int start_pos; /* The character position that defines the start of the input */ |
132 | | - int end_pos; /* The character position that defines the end of the input */ |
133 | | - position current; /* The current position */ |
134 | | - position start; /* The start position of the current token */ |
135 | | - bool first_token_of_line; /* This flag is used for tLINECOMMENT */ |
136 | | - unsigned int last_char; /* Last peeked character */ |
| 131 | + int start_pos; /* The character position that defines the start of the input */ |
| 132 | + int end_pos; /* The character position that defines the end of the input */ |
| 133 | + position current; /* The current position */ |
| 134 | + position start; /* The start position of the current token */ |
| 135 | + bool first_token_of_line; /* This flag is used for tLINECOMMENT */ |
| 136 | + unsigned int last_char; /* Last peeked character */ |
137 | 137 | } lexstate; |
138 | 138 |
|
139 | 139 | extern token NullToken; |
|
0 commit comments