8 * when in state state, and one of the characters
9 * in ch arrives, enter nextstate.
10 * States >= S_SELF are either final, or at least require special action.
11 * In 'fsm' there is a line for each state X charset X nextstate.
12 * List chars that overwrite previous entries later (e.g. C_ALPH
13 * can be overridden by '_' by a later entry; and C_XX is the
14 * the universal set, and should always be first.
15 * States above S_SELF are represented in the big table as negative values.
16 * S_SELF and S_SELFB encode the resulting token type in the upper bits.
17 * These actions differ in that S_SELF doesn't have a lookahead char,
20 * The encoding is blown out into a big table for time-efficiency.
22 * nextstate: 6 bits; ?\ marker: 1 bit; tokentype: 9 bits.
26 #define ACT(tok,act) ((tok<<7)+act)
28 #define GETACT(st) (st>>7)&0x1ff
30 #define UTF2(c) ((c)>=0xA0 && (c)<0xE0) /* 2-char UTF seq */
31 #define UTF3(c) ((c)>=0xE0 && (c)<0xF0) /* 3-char UTF seq */
32 #define UTF4(c) ((c)>=0xF0 && (c)<0xF8) /* 4-char UTF seq */
34 /* character classes */
42 START=0, NUM1, NUM2, NUM3, ID1, ST1, ST2, ST3, COM1, COM2, COM3, COM4,
43 CC1, CC2, WS1, PLUS1, MINUS1, STAR1, SLASH1, PCT1, SHARP1,
44 CIRC1, GT1, GT2, LT1, LT2, OR1, AND1, ASG1, NOT1, DOTS1,
45 S_SELF=MAXSTATE, S_SELFB, S_EOF, S_NL, S_EOFSTR,
46 S_STNL, S_COMNL, S_EOFCOM, S_COMMENT, S_WS, S_NAME
52 int state; /* if in this state */
53 uchar ch[4]; /* and see one of these characters */
54 int nextstate; /* enter this state if +ve */
57 /*const*/ struct fsm fsm[] = {
59 START, { C_XX }, ACT(UNCLASS,S_SELF),
60 START, { ' ', '\t', '\v', '\r' }, WS1,
61 START, { C_NUM }, NUM1,
63 START, { C_ALPH }, ID1,
68 START, { EOFC }, S_EOF,
69 START, { '\n' }, S_NL,
70 START, { '-' }, MINUS1,
71 START, { '+' }, PLUS1,
78 START, { '#' }, SHARP1,
80 START, { '[' }, ACT(SBRA,S_SELF),
81 START, { ']' }, ACT(SKET,S_SELF),
82 START, { '(' }, ACT(LP,S_SELF),
83 START, { ')' }, ACT(RP,S_SELF),
84 START, { '*' }, STAR1,
85 START, { ',' }, ACT(COMMA,S_SELF),
86 START, { '?' }, ACT(QUEST,S_SELF),
87 START, { ':' }, ACT(COLON,S_SELF),
88 START, { ';' }, ACT(SEMIC,S_SELF),
89 START, { '{' }, ACT(CBRA,S_SELF),
90 START, { '}' }, ACT(CKET,S_SELF),
91 START, { '~' }, ACT(TILDE,S_SELF),
92 START, { '^' }, CIRC1,
95 NUM1, { C_XX }, ACT(NUMBER,S_SELFB),
96 NUM1, { C_NUM, C_ALPH, '.' }, NUM1,
97 NUM1, { 'E', 'e' }, NUM2,
98 NUM1, { '_' }, ACT(NUMBER,S_SELFB),
100 /* saw possible start of exponent, digits-e */
101 NUM2, { C_XX }, ACT(NUMBER,S_SELFB),
102 NUM2, { '+', '-' }, NUM1,
103 NUM2, { C_NUM, C_ALPH }, NUM1,
104 NUM2, { '_' }, ACT(NUMBER,S_SELFB),
106 /* saw a '.', which could be a number or an operator */
107 NUM3, { C_XX }, ACT(DOT,S_SELFB),
108 NUM3, { '.' }, DOTS1,
109 NUM3, { C_NUM }, NUM1,
111 DOTS1, { C_XX }, ACT(UNCLASS, S_SELFB),
112 DOTS1, { C_NUM }, NUM1,
113 DOTS1, { '.' }, ACT(ELLIPS, S_SELF),
115 /* saw a letter or _ */
116 ID1, { C_XX }, ACT(NAME,S_NAME),
117 ID1, { C_ALPH, C_NUM }, ID1,
119 /* saw L (start of wide string?) */
120 ST1, { C_XX }, ACT(NAME,S_NAME),
121 ST1, { C_ALPH, C_NUM }, ID1,
125 /* saw " beginning string */
127 ST2, { '"' }, ACT(STRING, S_SELF),
129 ST2, { '\n' }, S_STNL,
130 ST2, { EOFC }, S_EOFSTR,
132 /* saw \ in string */
134 ST3, { '\n' }, S_STNL,
135 ST3, { EOFC }, S_EOFSTR,
137 /* saw ' beginning character const */
139 CC1, { '\'' }, ACT(CCON, S_SELF),
141 CC1, { '\n' }, S_STNL,
142 CC1, { EOFC }, S_EOFSTR,
146 CC2, { '\n' }, S_STNL,
147 CC2, { EOFC }, S_EOFSTR,
149 /* saw /, perhaps start of comment */
150 COM1, { C_XX }, ACT(SLASH, S_SELFB),
151 COM1, { '=' }, ACT(ASSLASH, S_SELF),
155 /* saw "/*", start of comment */
156 COM2, { C_XX }, COM2,
157 COM2, { '\n' }, S_COMNL,
159 COM2, { EOFC }, S_EOFCOM,
161 /* saw the * possibly ending a comment */
162 COM3, { C_XX }, COM2,
163 COM3, { '\n' }, S_COMNL,
165 COM3, { '/' }, S_COMMENT,
166 COM3, { EOFC }, S_EOFCOM,
169 COM4, { C_XX }, COM4,
170 COM4, { '\n' }, S_NL,
171 COM4, { EOFC }, S_EOFCOM,
173 /* saw white space, eat it up */
175 WS1, { ' ', '\t', '\v', '\r'}, WS1,
177 /* saw -, check --, -=, -> */
178 MINUS1, { C_XX }, ACT(MINUS, S_SELFB),
179 MINUS1, { '-' }, ACT(MMINUS, S_SELF),
180 MINUS1, { '=' }, ACT(ASMINUS,S_SELF),
181 MINUS1, { '>' }, ACT(ARROW,S_SELF),
183 /* saw +, check ++, += */
184 PLUS1, { C_XX }, ACT(PLUS, S_SELFB),
185 PLUS1, { '+' }, ACT(PPLUS, S_SELF),
186 PLUS1, { '=' }, ACT(ASPLUS, S_SELF),
188 /* saw <, check <<, <<=, <= */
189 LT1, { C_XX }, ACT(LT, S_SELFB),
191 LT1, { '=' }, ACT(LEQ, S_SELF),
192 LT2, { C_XX }, ACT(LSH, S_SELFB),
193 LT2, { '=' }, ACT(ASLSH, S_SELF),
195 /* saw >, check >>, >>=, >= */
196 GT1, { C_XX }, ACT(GT, S_SELFB),
198 GT1, { '=' }, ACT(GEQ, S_SELF),
199 GT2, { C_XX }, ACT(RSH, S_SELFB),
200 GT2, { '=' }, ACT(ASRSH, S_SELF),
203 ASG1, { C_XX }, ACT(ASGN, S_SELFB),
204 ASG1, { '=' }, ACT(EQ, S_SELF),
207 NOT1, { C_XX }, ACT(NOT, S_SELFB),
208 NOT1, { '=' }, ACT(NEQ, S_SELF),
211 AND1, { C_XX }, ACT(AND, S_SELFB),
212 AND1, { '&' }, ACT(LAND, S_SELF),
213 AND1, { '=' }, ACT(ASAND, S_SELF),
216 OR1, { C_XX }, ACT(OR, S_SELFB),
217 OR1, { '|' }, ACT(LOR, S_SELF),
218 OR1, { '=' }, ACT(ASOR, S_SELF),
221 SHARP1, { C_XX }, ACT(SHARP, S_SELFB),
222 SHARP1, { '#' }, ACT(DSHARP, S_SELF),
225 PCT1, { C_XX }, ACT(PCT, S_SELFB),
226 PCT1, { '=' }, ACT(ASPCT, S_SELF),
229 STAR1, { C_XX }, ACT(STAR, S_SELFB),
230 STAR1, { '=' }, ACT(ASSTAR, S_SELF),
233 CIRC1, { C_XX }, ACT(CIRC, S_SELFB),
234 CIRC1, { '=' }, ACT(ASCIRC, S_SELF),
239 /* first index is char, second is state */
240 /* increase #states to power of 2 to encourage use of shift */
241 short bigfsm[256][MAXSTATE];
246 /*const*/ struct fsm *fp;
249 for (fp = fsm; fp->state>=0; fp++) {
250 for (i=0; fp->ch[i]; i++) {
251 nstate = fp->nextstate;
252 if (nstate >= S_SELF)
256 case C_XX: /* random characters */
257 for (j=0; j<256; j++)
258 bigfsm[j][fp->state] = nstate;
261 for (j=0; j<=256; j++)
262 if ('a'<=j&&j<='z' || 'A'<=j&&j<='Z'
263 || UTF2(j) || UTF3(j) || UTF4(j) || j=='_')
264 bigfsm[j][fp->state] = nstate;
267 for (j='0'; j<='9'; j++)
268 bigfsm[j][fp->state] = nstate;
271 bigfsm[fp->ch[i]][fp->state] = nstate;
275 /* install special cases for ? (trigraphs), \ (splicing), runes */
276 for (i=0; i<MAXSTATE; i++) {
277 for (j=0; j<0xFF; j++)
278 if (j=='?' || j=='\\' || UTF2(j) || UTF3(j) || UTF4(j)) {
280 bigfsm[j][i] = ~bigfsm[j][i];
281 bigfsm[j][i] &= ~QBSBIT;
283 if (bigfsm[EOFC][i]>=0)
284 bigfsm[EOFC][i] = ~S_EOF;
291 /* do C++ comments? */
293 bigfsm['/'][COM1] = bigfsm['x'][COM1];
297 * fill in a row of tokens from input, terminated by NL or END
298 * First token is put at trp->lp.
299 * Reset is non-zero when the input buffer can be "rewound."
300 * The value is a flag indicating that possible macros have
301 * been seen in the row.
304 gettokens(Tokenrow *trp, int reset)
306 register int c, state, oldstate;
308 register Token *tp, *maxp;
310 Source *s = cursource;
317 maxp = &trp->bp[trp->max];
323 tp = growtokenrow(trp);
324 maxp = &trp->bp[trp->max];
335 if ((state = bigfsm[c][state]) >= 0) {
342 switch (state&0177) {
347 tp->type = GETACT(state);
348 tp->len = ip - tp->t;
352 case S_NAME: /* like S_SELFB but with nmac check */
354 tp->len = ip - tp->t;
355 nmac |= quicklook(tp->t[0], tp->len>1?tp->t[1]:0);
360 tp->wslen = ip - tp->t;
366 if ((state&QBSBIT)==0) {
373 if (c=='?') { /* check trigraph */
380 if (c=='\\') { /* line-folding */
400 error(WARNING, "Lexical botch in cpp");
409 if (tp!=trp->bp && (tp-1)->type!=NL && cursource->fd!=-1)
410 error(WARNING,"No newline at end of file");
415 error(ERROR, "Unterminated string or char const");
427 error(FATAL, "EOF in string or char constant");
438 error(WARNING, "EOF inside comment");
452 tp->len = ip - tp->t;
457 /* have seen ?; handle the trigraph it starts (if any) else 0 */
488 memmove(s->inp+1, s->inp+3, s->inl-s->inp+2);
500 if (s->inp[ncr+1] == '\r') { /* nonstandardly, ignore CR before line-folding */
504 if (s->inp[ncr+1] == '\n') {
505 memmove(s->inp, s->inp+2+ncr, s->inl-s->inp+3-ncr);
513 * Push down to new source of characters.
514 * If fd>0 and str==NULL, then from a file `name';
515 * if fd==-1 and str, then from the string.
518 setsource(char *name, int fd, char *str)
520 Source *s = new(Source);
530 /* slop at right for EOFC */
533 s->inb = domalloc(len+4);
534 strncpy((char *)s->inb, str, len);
539 s->inb = dorealloc(s->inb, len + INS);
540 if (s->fd<0 || (n=read(s->fd, (char *)s->inb + len, INS)) <= 0)
544 s->inb = dorealloc(s->inb, len + 4);
548 s->inl[0] = s->inl[1] = s->inl[2] = s->inl[3] = EOFC;
555 Source *s = cursource;