sys/src/cmd/cpp/lex.c

   1 #include <u.h>
   2 #include <libc.h>
   3 #include <stdio.h>
   4 #include "cpp.h"
   5
   6 /*
   7  * lexical FSM encoding
   8  *   when in state state, and one of the characters
   9  *   in ch arrives, enter nextstate.
  10  *   States >= S_SELF are either final, or at least require special action.
  11  *   In 'fsm' there is a line for each state X charset X nextstate.
  12  *   List chars that overwrite previous entries later (e.g. C_ALPH
  13  *   can be overridden by '_' by a later entry; and C_XX is the
  14  *   the universal set, and should always be first.
  15  *   States above S_SELF are represented in the big table as negative values.
  16  *   S_SELF and S_SELFB encode the resulting token type in the upper bits.
  17  *   These actions differ in that S_SELF doesn't have a lookahead char,
  18  *   S_SELFB does.
  19  *
  20  *   The encoding is blown out into a big table for time-efficiency.
  21  *   Entries have
  22  *      nextstate: 6 bits; ?\ marker: 1 bit; tokentype: 9 bits.
  23  */
  24
  25 #define MAXSTATE 32
  26 #define ACT(tok,act)    ((tok<<7)+act)
  27 #define QBSBIT  0100
  28 #define GETACT(st)      (st>>7)&0x1ff
  29
  30 #define UTF2(c)         ((c)>=0xA0 && (c)<0xE0)         /* 2-char UTF seq */
  31 #define UTF3(c)         ((c)>=0xE0 && (c)<0xF0)         /* 3-char UTF seq */
  32 #define UTF4(c)         ((c)>=0xF0 && (c)<0xF8)         /* 4-char UTF seq */
  33
  34 /* character classes */
  35 #define C_WS    1
  36 #define C_ALPH  2
  37 #define C_NUM   3
  38 #define C_EOF   4
  39 #define C_XX    5
  40
  41 enum state {
  42         START=0, NUM1, NUM2, NUM3, ID1, ST1, ST2, ST3, COM1, COM2, COM3, COM4,
  43         CC1, CC2, WS1, PLUS1, MINUS1, STAR1, SLASH1, PCT1, SHARP1,
  44         CIRC1, GT1, GT2, LT1, LT2, OR1, AND1, ASG1, NOT1, DOTS1,
  45         S_SELF=MAXSTATE, S_SELFB, S_EOF, S_NL, S_EOFSTR,
  46         S_STNL, S_COMNL, S_EOFCOM, S_COMMENT, S_WS, S_NAME
  47 };
  48
  49 int     tottok;
  50 int     tokkind[256];
  51 struct  fsm {
  52         int     state;          /* if in this state */
  53         uchar   ch[4];          /* and see one of these characters */
  54         int     nextstate;      /* enter this state if +ve */
  55 };
  56
  57 /*const*/ struct fsm fsm[] = {
  58         /* start state */
  59         START,  { C_XX },       ACT(UNCLASS,S_SELF),
  60         START,  { ' ', '\t', '\v', '\r' },      WS1,
  61         START,  { C_NUM },      NUM1,
  62         START,  { '.' },        NUM3,
  63         START,  { C_ALPH },     ID1,
  64         START,  { 'L' },        ST1,
  65         START,  { '"' },        ST2,
  66         START,  { '\'' },       CC1,
  67         START,  { '/' },        COM1,
  68         START,  { EOFC },       S_EOF,
  69         START,  { '\n' },       S_NL,
  70         START,  { '-' },        MINUS1,
  71         START,  { '+' },        PLUS1,
  72         START,  { '<' },        LT1,
  73         START,  { '>' },        GT1,
  74         START,  { '=' },        ASG1,
  75         START,  { '!' },        NOT1,
  76         START,  { '&' },        AND1,
  77         START,  { '|' },        OR1,
  78         START,  { '#' },        SHARP1,
  79         START,  { '%' },        PCT1,
  80         START,  { '[' },        ACT(SBRA,S_SELF),
  81         START,  { ']' },        ACT(SKET,S_SELF),
  82         START,  { '(' },        ACT(LP,S_SELF),
  83         START,  { ')' },        ACT(RP,S_SELF),
  84         START,  { '*' },        STAR1,
  85         START,  { ',' },        ACT(COMMA,S_SELF),
  86         START,  { '?' },        ACT(QUEST,S_SELF),
  87         START,  { ':' },        ACT(COLON,S_SELF),
  88         START,  { ';' },        ACT(SEMIC,S_SELF),
  89         START,  { '{' },        ACT(CBRA,S_SELF),
  90         START,  { '}' },        ACT(CKET,S_SELF),
  91         START,  { '~' },        ACT(TILDE,S_SELF),
  92         START,  { '^' },        CIRC1,
  93
  94         /* saw a digit */
  95         NUM1,   { C_XX },       ACT(NUMBER,S_SELFB),
  96         NUM1,   { C_NUM, C_ALPH, '.' }, NUM1,
  97         NUM1,   { 'E', 'e' },   NUM2,
  98         NUM1,   { '_' },        ACT(NUMBER,S_SELFB),
  99
 100         /* saw possible start of exponent, digits-e */
 101         NUM2,   { C_XX },       ACT(NUMBER,S_SELFB),
 102         NUM2,   { '+', '-' },   NUM1,
 103         NUM2,   { C_NUM, C_ALPH },      NUM1,
 104         NUM2,   { '_' },        ACT(NUMBER,S_SELFB),
 105
 106         /* saw a '.', which could be a number or an operator */
 107         NUM3,   { C_XX },       ACT(DOT,S_SELFB),
 108         NUM3,   { '.' },        DOTS1,
 109         NUM3,   { C_NUM },      NUM1,
 110
 111         DOTS1,  { C_XX },       ACT(UNCLASS, S_SELFB),
 112         DOTS1,  { C_NUM },      NUM1,
 113         DOTS1,  { '.' },        ACT(ELLIPS, S_SELF),
 114
 115         /* saw a letter or _ */
 116         ID1,    { C_XX },       ACT(NAME,S_NAME),
 117         ID1,    { C_ALPH, C_NUM },      ID1,
 118
 119         /* saw L (start of wide string?) */
 120         ST1,    { C_XX },       ACT(NAME,S_NAME),
 121         ST1,    { C_ALPH, C_NUM },      ID1,
 122         ST1,    { '"' },        ST2,
 123         ST1,    { '\'' },       CC1,
 124
 125         /* saw " beginning string */
 126         ST2,    { C_XX },       ST2,
 127         ST2,    { '"' },        ACT(STRING, S_SELF),
 128         ST2,    { '\\' },       ST3,
 129         ST2,    { '\n' },       S_STNL,
 130         ST2,    { EOFC },       S_EOFSTR,
 131
 132         /* saw \ in string */
 133         ST3,    { C_XX },       ST2,
 134         ST3,    { '\n' },       S_STNL,
 135         ST3,    { EOFC },       S_EOFSTR,
 136
 137         /* saw ' beginning character const */
 138         CC1,    { C_XX },       CC1,
 139         CC1,    { '\'' },       ACT(CCON, S_SELF),
 140         CC1,    { '\\' },       CC2,
 141         CC1,    { '\n' },       S_STNL,
 142         CC1,    { EOFC },       S_EOFSTR,
 143
 144         /* saw \ in ccon */
 145         CC2,    { C_XX },       CC1,
 146         CC2,    { '\n' },       S_STNL,
 147         CC2,    { EOFC },       S_EOFSTR,
 148
 149         /* saw /, perhaps start of comment */
 150         COM1,   { C_XX },       ACT(SLASH, S_SELFB),
 151         COM1,   { '=' },        ACT(ASSLASH, S_SELF),
 152         COM1,   { '*' },        COM2,
 153         COM1,   { '/' },        COM4,
 154
 155         /* saw "/*", start of comment */
 156         COM2,   { C_XX },       COM2,
 157         COM2,   { '\n' },       S_COMNL,
 158         COM2,   { '*' },        COM3,
 159         COM2,   { EOFC },       S_EOFCOM,
 160
 161         /* saw the * possibly ending a comment */
 162         COM3,   { C_XX },       COM2,
 163         COM3,   { '\n' },       S_COMNL,
 164         COM3,   { '*' },        COM3,
 165         COM3,   { '/' },        S_COMMENT,
 166         COM3,   { EOFC },       S_EOFCOM,
 167
 168         /* // comment */
 169         COM4,   { C_XX },       COM4,
 170         COM4,   { '\n' },       S_NL,
 171         COM4,   { EOFC },       S_EOFCOM,
 172
 173         /* saw white space, eat it up */
 174         WS1,    { C_XX },       S_WS,
 175         WS1,    { ' ', '\t', '\v', '\r'},       WS1,
 176
 177         /* saw -, check --, -=, -> */
 178         MINUS1, { C_XX },       ACT(MINUS, S_SELFB),
 179         MINUS1, { '-' },        ACT(MMINUS, S_SELF),
 180         MINUS1, { '=' },        ACT(ASMINUS,S_SELF),
 181         MINUS1, { '>' },        ACT(ARROW,S_SELF),
 182
 183         /* saw +, check ++, += */
 184         PLUS1,  { C_XX },       ACT(PLUS, S_SELFB),
 185         PLUS1,  { '+' },        ACT(PPLUS, S_SELF),
 186         PLUS1,  { '=' },        ACT(ASPLUS, S_SELF),
 187
 188         /* saw <, check <<, <<=, <= */
 189         LT1,    { C_XX },       ACT(LT, S_SELFB),
 190         LT1,    { '<' },        LT2,
 191         LT1,    { '=' },        ACT(LEQ, S_SELF),
 192         LT2,    { C_XX },       ACT(LSH, S_SELFB),
 193         LT2,    { '=' },        ACT(ASLSH, S_SELF),
 194
 195         /* saw >, check >>, >>=, >= */
 196         GT1,    { C_XX },       ACT(GT, S_SELFB),
 197         GT1,    { '>' },        GT2,
 198         GT1,    { '=' },        ACT(GEQ, S_SELF),
 199         GT2,    { C_XX },       ACT(RSH, S_SELFB),
 200         GT2,    { '=' },        ACT(ASRSH, S_SELF),
 201
 202         /* = */
 203         ASG1,   { C_XX },       ACT(ASGN, S_SELFB),
 204         ASG1,   { '=' },        ACT(EQ, S_SELF),
 205
 206         /* ! */
 207         NOT1,   { C_XX },       ACT(NOT, S_SELFB),
 208         NOT1,   { '=' },        ACT(NEQ, S_SELF),
 209
 210         /* & */
 211         AND1,   { C_XX },       ACT(AND, S_SELFB),
 212         AND1,   { '&' },        ACT(LAND, S_SELF),
 213         AND1,   { '=' },        ACT(ASAND, S_SELF),
 214
 215         /* | */
 216         OR1,    { C_XX },       ACT(OR, S_SELFB),
 217         OR1,    { '|' },        ACT(LOR, S_SELF),
 218         OR1,    { '=' },        ACT(ASOR, S_SELF),
 219
 220         /* # */
 221         SHARP1, { C_XX },       ACT(SHARP, S_SELFB),
 222         SHARP1, { '#' },        ACT(DSHARP, S_SELF),
 223
 224         /* % */
 225         PCT1,   { C_XX },       ACT(PCT, S_SELFB),
 226         PCT1,   { '=' },        ACT(ASPCT, S_SELF),
 227
 228         /* * */
 229         STAR1,  { C_XX },       ACT(STAR, S_SELFB),
 230         STAR1,  { '=' },        ACT(ASSTAR, S_SELF),
 231
 232         /* ^ */
 233         CIRC1,  { C_XX },       ACT(CIRC, S_SELFB),
 234         CIRC1,  { '=' },        ACT(ASCIRC, S_SELF),
 235
 236         -1
 237 };
 238
 239 /* first index is char, second is state */
 240 /* increase #states to power of 2 to encourage use of shift */
 241 short   bigfsm[256][MAXSTATE];
 242
 243 void
 244 expandlex(void)
 245 {
 246         /*const*/ struct fsm *fp;
 247         int i, j, nstate;
 248
 249         for (fp = fsm; fp->state>=0; fp++) {
 250                 for (i=0; fp->ch[i]; i++) {
 251                         nstate = fp->nextstate;
 252                         if (nstate >= S_SELF)
 253                                 nstate = ~nstate;
 254                         switch (fp->ch[i]) {
 255
 256                         case C_XX:              /* random characters */
 257                                 for (j=0; j<256; j++)
 258                                         bigfsm[j][fp->state] = nstate;
 259                                 continue;
 260                         case C_ALPH:
 261                                 for (j=0; j<=256; j++)
 262                                         if ('a'<=j&&j<='z' || 'A'<=j&&j<='Z'
 263                                           || UTF2(j) || UTF3(j) || UTF4(j) || j=='_')
 264                                                 bigfsm[j][fp->state] = nstate;
 265                                 continue;
 266                         case C_NUM:
 267                                 for (j='0'; j<='9'; j++)
 268                                         bigfsm[j][fp->state] = nstate;
 269                                 continue;
 270                         default:
 271                                 bigfsm[fp->ch[i]][fp->state] = nstate;
 272                         }
 273                 }
 274         }
 275         /* install special cases for ? (trigraphs),  \ (splicing), runes */
 276         for (i=0; i<MAXSTATE; i++) {
 277                 for (j=0; j<0xFF; j++)
 278                         if (j=='?' || j=='\\' || UTF2(j) || UTF3(j) || UTF4(j)) {
 279                                 if (bigfsm[j][i]>0)
 280                                         bigfsm[j][i] = ~bigfsm[j][i];
 281                                 bigfsm[j][i] &= ~QBSBIT;
 282                         }
 283                 if (bigfsm[EOFC][i]>=0)
 284                         bigfsm[EOFC][i] = ~S_EOF;
 285         }
 286 }
 287
 288 void
 289 fixlex(void)
 290 {
 291         /* do C++ comments? */
 292         if (Cplusplus==0)
 293                 bigfsm['/'][COM1] = bigfsm['x'][COM1];
 294 }
 295
 296 /*
 297  * fill in a row of tokens from input, terminated by NL or END
 298  * First token is put at trp->lp.
 299  * Reset is non-zero when the input buffer can be "rewound."
 300  * The value is a flag indicating that possible macros have
 301  * been seen in the row.
 302  */
 303 int
 304 gettokens(Tokenrow *trp, int reset)
 305 {
 306         register int c, state, oldstate;
 307         register uchar *ip;
 308         register Token *tp, *maxp;
 309         int runelen;
 310         Source *s = cursource;
 311         int nmac = 0;
 312
 313         tp = trp->lp;
 314         ip = s->inp;
 315         if (reset)
 316                 s->lineinc = 0;
 317         maxp = &trp->bp[trp->max];
 318         runelen = 1;
 319         for (;;) {
 320            continue2:
 321                 if (tp>=maxp) {
 322                         trp->lp = tp;
 323                         tp = growtokenrow(trp);
 324                         maxp = &trp->bp[trp->max];
 325                 }
 326                 tp->type = UNCLASS;
 327                 tp->hideset = 0;
 328                 tp->t = ip;
 329                 tp->wslen = 0;
 330                 tp->flag = 0;
 331                 state = START;
 332                 for (;;) {
 333                         oldstate = state;
 334                         c = *ip;
 335                         if ((state = bigfsm[c][state]) >= 0) {
 336                                 ip += runelen;
 337                                 runelen = 1;
 338                                 continue;
 339                         }
 340                         state = ~state;
 341                 reswitch:
 342                         switch (state&0177) {
 343                         case S_SELF:
 344                                 ip += runelen;
 345                                 runelen = 1;
 346                         case S_SELFB:
 347                                 tp->type = GETACT(state);
 348                                 tp->len = ip - tp->t;
 349                                 tp++;
 350                                 goto continue2;
 351
 352                         case S_NAME:    /* like S_SELFB but with nmac check */
 353                                 tp->type = NAME;
 354                                 tp->len = ip - tp->t;
 355                                 nmac |= quicklook(tp->t[0], tp->len>1?tp->t[1]:0);
 356                                 tp++;
 357                                 goto continue2;
 358
 359                         case S_WS:
 360                                 tp->wslen = ip - tp->t;
 361                                 tp->t = ip;
 362                                 state = START;
 363                                 continue;
 364
 365                         default:
 366                                 if ((state&QBSBIT)==0) {
 367                                         ip += runelen;
 368                                         runelen = 1;
 369                                         continue;
 370                                 }
 371                                 state &= ~QBSBIT;
 372                                 s->inp = ip;
 373                                 if (c=='?') {   /* check trigraph */
 374                                         if (trigraph(s)) {
 375                                                 state = oldstate;
 376                                                 continue;
 377                                         }
 378                                         goto reswitch;
 379                                 }
 380                                 if (c=='\\') { /* line-folding */
 381                                         if (foldline(s)) {
 382                                                 s->lineinc++;
 383                                                 state = oldstate;
 384                                                 continue;
 385                                         }
 386                                         goto reswitch;
 387                                 }
 388                                 if (UTF2(c)) {
 389                                         runelen = 2;
 390                                         goto reswitch;
 391                                 }
 392                                 if (UTF3(c)) {
 393                                         runelen = 3;
 394                                         goto reswitch;
 395                                 }
 396                                 if (UTF4(c)) {
 397                                         runelen = 4;
 398                                         goto reswitch;
 399                                 }
 400                                 error(WARNING, "Lexical botch in cpp");
 401                                 ip += runelen;
 402                                 runelen = 1;
 403                                 continue;
 404
 405                         case S_EOF:
 406                                 tp->type = END;
 407                                 tp->len = 0;
 408                                 s->inp = ip;
 409                                 if (tp!=trp->bp && (tp-1)->type!=NL && cursource->fd!=-1)
 410                                         error(WARNING,"No newline at end of file");
 411                                 trp->lp = tp+1;
 412                                 return nmac;
 413
 414                         case S_STNL:
 415                                 error(ERROR, "Unterminated string or char const");
 416                         case S_NL:
 417                                 tp->t = ip;
 418                                 tp->type = NL;
 419                                 tp->len = 1;
 420                                 tp->wslen = 0;
 421                                 s->lineinc++;
 422                                 s->inp = ip+1;
 423                                 trp->lp = tp+1;
 424                                 return nmac;
 425
 426                         case S_EOFSTR:
 427                                 error(FATAL, "EOF in string or char constant");
 428                                 break;
 429
 430                         case S_COMNL:
 431                                 s->lineinc++;
 432                                 state = COM2;
 433                                 ip += runelen;
 434                                 runelen = 1;
 435                                 continue;
 436
 437                         case S_EOFCOM:
 438                                 error(WARNING, "EOF inside comment");
 439                                 --ip;
 440                         case S_COMMENT:
 441                                 ++ip;
 442                                 tp->t = ip;
 443                                 tp->t[-1] = ' ';
 444                                 tp->wslen = 1;
 445                                 state = START;
 446                                 continue;
 447                         }
 448                         break;
 449                 }
 450                 ip += runelen;
 451                 runelen = 1;
 452                 tp->len = ip - tp->t;
 453                 tp++;
 454         }
 455 }
 456
 457 /* have seen ?; handle the trigraph it starts (if any) else 0 */
 458 int
 459 trigraph(Source *s)
 460 {
 461         int c;
 462
 463         if (s->inp[1]!='?')
 464                 return 0;
 465         c = 0;
 466         switch(s->inp[2]) {
 467         case '=':
 468                 c = '#'; break;
 469         case '(':
 470                 c = '['; break;
 471         case '/':
 472                 c = '\\'; break;
 473         case ')':
 474                 c = ']'; break;
 475         case '\'':
 476                 c = '^'; break;
 477         case '<':
 478                 c = '{'; break;
 479         case '!':
 480                 c = '|'; break;
 481         case '>':
 482                 c = '}'; break;
 483         case '-':
 484                 c = '~'; break;
 485         }
 486         if (c) {
 487                 *s->inp = c;
 488                 memmove(s->inp+1, s->inp+3, s->inl-s->inp+2);
 489                 s->inl -= 2;
 490         }
 491         return c;
 492 }
 493
 494 int
 495 foldline(Source *s)
 496 {
 497         int ncr = 0;
 498
 499 recheck:
 500         if (s->inp[ncr+1] == '\r') {    /* nonstandardly, ignore CR before line-folding */
 501                 ncr++;
 502                 goto recheck;
 503         }
 504         if (s->inp[ncr+1] == '\n') {
 505                 memmove(s->inp, s->inp+2+ncr, s->inl-s->inp+3-ncr);
 506                 s->inl -= 2+ncr;
 507                 return 1;
 508         }
 509         return 0;
 510 }
 511
 512 /*
 513  * Push down to new source of characters.
 514  * If fd>0 and str==NULL, then from a file `name';
 515  * if fd==-1 and str, then from the string.
 516  */
 517 Source *
 518 setsource(char *name, int fd, char *str)
 519 {
 520         Source *s = new(Source);
 521         int n, len;
 522
 523         s->line = 1;
 524         s->lineinc = 0;
 525         s->fd = fd;
 526         s->filename = name;
 527         s->next = cursource;
 528         s->ifdepth = 0;
 529         cursource = s;
 530         /* slop at right for EOFC */
 531         if (str) {
 532                 len = strlen(str);
 533                 s->inb = domalloc(len+4);
 534                 strncpy((char *)s->inb, str, len);
 535         } else {
 536                 len = 0;
 537                 s->inb = nil;
 538                 for(;;){
 539                         s->inb = dorealloc(s->inb, len + INS);
 540                         if (s->fd<0 || (n=read(s->fd, (char *)s->inb + len, INS)) <= 0)
 541                                 break;
 542                         len += n;
 543                 }
 544                 s->inb = dorealloc(s->inb, len + 4);
 545         }
 546         s->inp = s->inb;
 547         s->inl = s->inp+len;
 548         s->inl[0] = s->inl[1] = s->inl[2] = s->inl[3] = EOFC;
 549         return s;
 550 }
 551
 552 void
 553 unsetsource(void)
 554 {
 555         Source *s = cursource;
 556
 557         if (s->fd>=0) {
 558                 close(s->fd);
 559                 dofree(s->inb);
 560         }
 561         cursource = s->next;
 562         dofree(s);
 563 }