8 * when in state state, and one of the characters
9 * in ch arrives, enter nextstate.
10 * States >= S_SELF are either final, or at least require special action.
11 * In 'fsm' there is a line for each state X charset X nextstate.
12 * List chars that overwrite previous entries later (e.g. C_ALPH
13 * can be overridden by '_' by a later entry; and C_XX is the
14 * the universal set, and should always be first.
15 * States above S_SELF are represented in the big table as negative values.
16 * S_SELF and S_SELFB encode the resulting token type in the upper bits.
17 * These actions differ in that S_SELF doesn't have a lookahead char,
20 * The encoding is blown out into a big table for time-efficiency.
22 * nextstate: 6 bits; ?\ marker: 1 bit; tokentype: 9 bits.
26 #define ACT(tok,act) ((tok<<7)+act)
28 #define GETACT(st) (st>>7)&0x1ff
30 #define UTF2(c) ((c)>=0xA0 && (c)<0xE0) /* 2-char UTF seq */
31 #define UTF3(c) ((c)>=0xE0 && (c)<0xF0) /* 3-char UTF seq */
33 /* character classes */
41 START=0, NUM1, NUM2, NUM3, ID1, ST1, ST2, ST3, COM1, COM2, COM3, COM4,
42 CC1, CC2, WS1, PLUS1, MINUS1, STAR1, SLASH1, PCT1, SHARP1,
43 CIRC1, GT1, GT2, LT1, LT2, OR1, AND1, ASG1, NOT1, DOTS1,
44 S_SELF=MAXSTATE, S_SELFB, S_EOF, S_NL, S_EOFSTR,
45 S_STNL, S_COMNL, S_EOFCOM, S_COMMENT, S_EOB, S_WS, S_NAME
51 int state; /* if in this state */
52 uchar ch[4]; /* and see one of these characters */
53 int nextstate; /* enter this state if +ve */
56 /*const*/ struct fsm fsm[] = {
58 START, { C_XX }, ACT(UNCLASS,S_SELF),
59 START, { ' ', '\t', '\v', '\r' }, WS1,
60 START, { C_NUM }, NUM1,
62 START, { C_ALPH }, ID1,
67 START, { EOFC }, S_EOF,
68 START, { '\n' }, S_NL,
69 START, { '-' }, MINUS1,
70 START, { '+' }, PLUS1,
77 START, { '#' }, SHARP1,
79 START, { '[' }, ACT(SBRA,S_SELF),
80 START, { ']' }, ACT(SKET,S_SELF),
81 START, { '(' }, ACT(LP,S_SELF),
82 START, { ')' }, ACT(RP,S_SELF),
83 START, { '*' }, STAR1,
84 START, { ',' }, ACT(COMMA,S_SELF),
85 START, { '?' }, ACT(QUEST,S_SELF),
86 START, { ':' }, ACT(COLON,S_SELF),
87 START, { ';' }, ACT(SEMIC,S_SELF),
88 START, { '{' }, ACT(CBRA,S_SELF),
89 START, { '}' }, ACT(CKET,S_SELF),
90 START, { '~' }, ACT(TILDE,S_SELF),
91 START, { '^' }, CIRC1,
94 NUM1, { C_XX }, ACT(NUMBER,S_SELFB),
95 NUM1, { C_NUM, C_ALPH, '.' }, NUM1,
96 NUM1, { 'E', 'e' }, NUM2,
97 NUM1, { '_' }, ACT(NUMBER,S_SELFB),
99 /* saw possible start of exponent, digits-e */
100 NUM2, { C_XX }, ACT(NUMBER,S_SELFB),
101 NUM2, { '+', '-' }, NUM1,
102 NUM2, { C_NUM, C_ALPH }, NUM1,
103 NUM2, { '_' }, ACT(NUMBER,S_SELFB),
105 /* saw a '.', which could be a number or an operator */
106 NUM3, { C_XX }, ACT(DOT,S_SELFB),
107 NUM3, { '.' }, DOTS1,
108 NUM3, { C_NUM }, NUM1,
110 DOTS1, { C_XX }, ACT(UNCLASS, S_SELFB),
111 DOTS1, { C_NUM }, NUM1,
112 DOTS1, { '.' }, ACT(ELLIPS, S_SELF),
114 /* saw a letter or _ */
115 ID1, { C_XX }, ACT(NAME,S_NAME),
116 ID1, { C_ALPH, C_NUM }, ID1,
118 /* saw L (start of wide string?) */
119 ST1, { C_XX }, ACT(NAME,S_NAME),
120 ST1, { C_ALPH, C_NUM }, ID1,
124 /* saw " beginning string */
126 ST2, { '"' }, ACT(STRING, S_SELF),
128 ST2, { '\n' }, S_STNL,
129 ST2, { EOFC }, S_EOFSTR,
131 /* saw \ in string */
133 ST3, { '\n' }, S_STNL,
134 ST3, { EOFC }, S_EOFSTR,
136 /* saw ' beginning character const */
138 CC1, { '\'' }, ACT(CCON, S_SELF),
140 CC1, { '\n' }, S_STNL,
141 CC1, { EOFC }, S_EOFSTR,
145 CC2, { '\n' }, S_STNL,
146 CC2, { EOFC }, S_EOFSTR,
148 /* saw /, perhaps start of comment */
149 COM1, { C_XX }, ACT(SLASH, S_SELFB),
150 COM1, { '=' }, ACT(ASSLASH, S_SELF),
154 /* saw "/*", start of comment */
155 COM2, { C_XX }, COM2,
156 COM2, { '\n' }, S_COMNL,
158 COM2, { EOFC }, S_EOFCOM,
160 /* saw the * possibly ending a comment */
161 COM3, { C_XX }, COM2,
162 COM3, { '\n' }, S_COMNL,
164 COM3, { '/' }, S_COMMENT,
165 COM3, { EOFC }, S_EOFCOM,
168 COM4, { C_XX }, COM4,
169 COM4, { '\n' }, S_NL,
170 COM4, { EOFC }, S_EOFCOM,
172 /* saw white space, eat it up */
174 WS1, { ' ', '\t', '\v', '\r'}, WS1,
176 /* saw -, check --, -=, -> */
177 MINUS1, { C_XX }, ACT(MINUS, S_SELFB),
178 MINUS1, { '-' }, ACT(MMINUS, S_SELF),
179 MINUS1, { '=' }, ACT(ASMINUS,S_SELF),
180 MINUS1, { '>' }, ACT(ARROW,S_SELF),
182 /* saw +, check ++, += */
183 PLUS1, { C_XX }, ACT(PLUS, S_SELFB),
184 PLUS1, { '+' }, ACT(PPLUS, S_SELF),
185 PLUS1, { '=' }, ACT(ASPLUS, S_SELF),
187 /* saw <, check <<, <<=, <= */
188 LT1, { C_XX }, ACT(LT, S_SELFB),
190 LT1, { '=' }, ACT(LEQ, S_SELF),
191 LT2, { C_XX }, ACT(LSH, S_SELFB),
192 LT2, { '=' }, ACT(ASLSH, S_SELF),
194 /* saw >, check >>, >>=, >= */
195 GT1, { C_XX }, ACT(GT, S_SELFB),
197 GT1, { '=' }, ACT(GEQ, S_SELF),
198 GT2, { C_XX }, ACT(RSH, S_SELFB),
199 GT2, { '=' }, ACT(ASRSH, S_SELF),
202 ASG1, { C_XX }, ACT(ASGN, S_SELFB),
203 ASG1, { '=' }, ACT(EQ, S_SELF),
206 NOT1, { C_XX }, ACT(NOT, S_SELFB),
207 NOT1, { '=' }, ACT(NEQ, S_SELF),
210 AND1, { C_XX }, ACT(AND, S_SELFB),
211 AND1, { '&' }, ACT(LAND, S_SELF),
212 AND1, { '=' }, ACT(ASAND, S_SELF),
215 OR1, { C_XX }, ACT(OR, S_SELFB),
216 OR1, { '|' }, ACT(LOR, S_SELF),
217 OR1, { '=' }, ACT(ASOR, S_SELF),
220 SHARP1, { C_XX }, ACT(SHARP, S_SELFB),
221 SHARP1, { '#' }, ACT(DSHARP, S_SELF),
224 PCT1, { C_XX }, ACT(PCT, S_SELFB),
225 PCT1, { '=' }, ACT(ASPCT, S_SELF),
228 STAR1, { C_XX }, ACT(STAR, S_SELFB),
229 STAR1, { '=' }, ACT(ASSTAR, S_SELF),
232 CIRC1, { C_XX }, ACT(CIRC, S_SELFB),
233 CIRC1, { '=' }, ACT(ASCIRC, S_SELF),
238 /* first index is char, second is state */
239 /* increase #states to power of 2 to encourage use of shift */
240 short bigfsm[256][MAXSTATE];
245 /*const*/ struct fsm *fp;
248 for (fp = fsm; fp->state>=0; fp++) {
249 for (i=0; fp->ch[i]; i++) {
250 nstate = fp->nextstate;
251 if (nstate >= S_SELF)
255 case C_XX: /* random characters */
256 for (j=0; j<256; j++)
257 bigfsm[j][fp->state] = nstate;
260 for (j=0; j<=256; j++)
261 if ('a'<=j&&j<='z' || 'A'<=j&&j<='Z'
262 || UTF2(j) || UTF3(j) || j=='_')
263 bigfsm[j][fp->state] = nstate;
266 for (j='0'; j<='9'; j++)
267 bigfsm[j][fp->state] = nstate;
270 bigfsm[fp->ch[i]][fp->state] = nstate;
274 /* install special cases for ? (trigraphs), \ (splicing), runes, and EOB */
275 for (i=0; i<MAXSTATE; i++) {
276 for (j=0; j<0xFF; j++)
277 if (j=='?' || j=='\\' || UTF2(j) || UTF3(j)) {
279 bigfsm[j][i] = ~bigfsm[j][i];
280 bigfsm[j][i] &= ~QBSBIT;
282 bigfsm[EOB][i] = ~S_EOB;
283 if (bigfsm[EOFC][i]>=0)
284 bigfsm[EOFC][i] = ~S_EOF;
291 /* do C++ comments? */
293 bigfsm['/'][COM1] = bigfsm['x'][COM1];
297 * fill in a row of tokens from input, terminated by NL or END
298 * First token is put at trp->lp.
299 * Reset is non-zero when the input buffer can be "rewound."
300 * The value is a flag indicating that possible macros have
301 * been seen in the row.
304 gettokens(Tokenrow *trp, int reset)
306 register int c, state, oldstate;
308 register Token *tp, *maxp;
310 Source *s = cursource;
312 extern char outbuf[];
318 if (ip>=s->inl) { /* nothing in buffer */
321 ip = s->inp = s->inb;
322 } else if (ip >= s->inb+(3*s->ins/4)) {
323 memmove(s->inb, ip, 4+s->inl-ip);
324 s->inl = s->inb+(s->inl-ip);
325 ip = s->inp = s->inb;
328 maxp = &trp->bp[trp->max];
334 tp = growtokenrow(trp);
335 maxp = &trp->bp[trp->max];
346 if ((state = bigfsm[c][state]) >= 0) {
353 switch (state&0177) {
358 tp->type = GETACT(state);
359 tp->len = ip - tp->t;
363 case S_NAME: /* like S_SELFB but with nmac check */
365 tp->len = ip - tp->t;
366 nmac |= quicklook(tp->t[0], tp->len>1?tp->t[1]:0);
371 tp->wslen = ip - tp->t;
377 if ((state&QBSBIT)==0) {
384 if (c=='?') { /* check trigraph */
391 if (c=='\\') { /* line-folding */
407 error(WARNING, "Lexical botch in cpp");
422 if (tp!=trp->bp && (tp-1)->type!=NL && cursource->fd!=-1)
423 error(WARNING,"No newline at end of file");
428 error(ERROR, "Unterminated string or char const");
440 error(FATAL, "EOF in string or char constant");
448 if (ip >= s->inb+(7*s->ins/8)) { /* very long comment */
449 memmove(tp->t, ip, 4+s->inl-ip);
456 error(WARNING, "EOF inside comment");
470 tp->len = ip - tp->t;
475 /* have seen ?; handle the trigraph it starts (if any) else 0 */
481 while (s->inp+2 >= s->inl && fillbuf(s)!=EOF)
508 memmove(s->inp+1, s->inp+3, s->inl-s->inp+2);
520 while (s->inp+1 >= s->inl && fillbuf(s)!=EOF)
522 if (s->inp[ncr+1] == '\r') { /* nonstandardly, ignore CR before line-folding */
526 if (s->inp[ncr+1] == '\n') {
527 memmove(s->inp, s->inp+2+ncr, s->inl-s->inp+3-ncr);
539 while((char *)s->inl+s->ins/8 > (char *)s->inb+s->ins) {
540 int l = s->inl - s->inb;
541 int p = s->inp - s->inb;
543 error(FATAL, "negative end of input!?");
545 error(FATAL, "negative input pointer!?");
546 /* double the buffer size and try again */
548 s->inb = dorealloc(s->inb, s->ins);
552 if (s->fd<0 || (n=read(s->fd, (char *)s->inl, s->ins/8)) <= 0)
554 if ((*s->inp&0xff) == EOB) /* sentinel character appears in input */
557 s->inl[0] = s->inl[1]= s->inl[2]= s->inl[3] = EOB;
559 s->inl[0] = s->inl[1]= s->inl[2]= s->inl[3] = EOFC;
566 * Push down to new source of characters.
567 * If fd>0 and str==NULL, then from a file `name';
568 * if fd==-1 and str, then from the string.
571 setsource(char *name, int fd, char *str)
573 Source *s = new(Source);
583 /* slop at right for EOB */
586 s->inb = domalloc(len+4);
588 strncpy((char *)s->inp, str, len);
601 s->inb = domalloc((junk)+4);
608 s->inl[0] = s->inl[1] = EOB;
615 Source *s = cursource;