Runeself = 0x80, /* rune and UTF sequences are the same (<) */
Runeerror = 0xFFFD, /* decoding error in UTF */
Runemax = 0xFFFF, /* 16 bit rune */
+ Runemask = 0xFFFF, /* bits used by runes (see grep) */
};
/*
typedef struct Inst Inst;
struct Inst
{
- uint type; /* < 0x10000 ==> literal, otherwise action */
+ uint type; /* <= Runemax+1 ==> literal, otherwise action */
union {
int sid;
int subid;
* 0x100xx are operators, value == precedence
* 0x200xx are tokens, i.e. operands for operators
*/
-#define OPERATOR 0x10000 /* Bitmask of all operators */
-#define START 0x10000 /* Start, used for marker on stack */
-#define RBRA 0x10001 /* Right bracket, ) */
-#define LBRA 0x10002 /* Left bracket, ( */
-#define OR 0x10003 /* Alternation, | */
-#define CAT 0x10004 /* Concatentation, implicit operator */
-#define STAR 0x10005 /* Closure, * */
-#define PLUS 0x10006 /* a+ == aa* */
-#define QUEST 0x10007 /* a? == a|nothing, i.e. 0 or 1 a's */
-#define ANY 0x20000 /* Any character but newline, . */
-#define NOP 0x20001 /* No operation, internal use only */
-#define BOL 0x20002 /* Beginning of line, ^ */
-#define EOL 0x20003 /* End of line, $ */
-#define CCLASS 0x20004 /* Character class, [] */
-#define NCCLASS 0x20005 /* Negated character class, [^] */
-#define END 0x20077 /* Terminate: match found */
-
-#define ISATOR 0x10000
-#define ISAND 0x20000
+enum {
+ OPERATOR = Runemask+1, /* Bitmask of all operators */
+ START = OPERATOR, /* Start, used for marker on stack */
+ RBRA, /* Right bracket, ) */
+ LBRA, /* Left bracket, ( */
+ OR, /* Alternation, | */
+ CAT, /* Concatentation, implicit operator */
+ STAR, /* Closure, * */
+ PLUS, /* a+ == aa* */
+ QUEST, /* a? == a|nothing, i.e. 0 or 1 a's */
+
+ ANY = OPERATOR<<1, /* Any character but newline, . */
+ NOP, /* No operation, internal use only */
+ BOL, /* Beginning of line, ^ */
+ EOL, /* End of line, $ */
+ CCLASS, /* Character class, [] */
+ NCCLASS, /* Negated character class, [^] */
+ END, /* Terminate: match found */
+
+ ISATOR = OPERATOR,
+ ISAND = OPERATOR<<1,
+};
/*
* Parser Information
exprp++;
return '\n';
}
- return *exprp++|0x10000;
+ return *exprp++|(Runemax+1);
}
return *exprp++;
}
ESIZE = 256, /* max size of reg exp */
GBSIZE = 256, /* max size of global command */
MAXSUB = 9, /* max number of sub reg exp */
- ESCFLG = 0xFFFF, /* escape Rune - user defined code */
+ ESCFLG = Runemax, /* escape Rune - user defined code */
EOF = -1,
};
if(c == 0)
continue;
*p++ = c;
- if(p >= &linebuf[LBSIZE-2])
+ if(p >= &linebuf[LBSIZE-sizeof(Rune)])
error(Q);
}
}
for(a1=addr1; a1<=addr2; a1++) {
lp = getline(*a1);
while(*gp = *lp++)
- if(gp++ >= &genbuf[LBSIZE-2])
+ if(gp++ >= &genbuf[LBSIZE-sizeof(Rune)])
error(Q);
}
lp = linebuf;
close(fd);
}
-/*
- * Unicode 4.0 4-byte runes.
- */
-typedef int Rune1;
-
-enum {
- UTFmax1 = 4,
-};
-
-int
-fullrune1(char *p, int n)
-{
- int c;
-
- if(n >= 1) {
- c = *(uchar*)p;
- if(c < 0x80)
- return 1;
- if(n >= 2 && c < 0xE0)
- return 1;
- if(n >= 3 && c < 0xF0)
- return 1;
- if(n >= 4)
- return 1;
- }
- return 0;
-}
-
-int
-chartorune1(Rune1 *rune, char *str)
-{
- int c, c1, c2, c3, n;
- Rune r;
-
- c = *(uchar*)str;
- if(c < 0xF0){
- r = 0;
- n = chartorune(&r, str);
- *rune = r;
- return n;
- }
- c &= ~0xF0;
- c1 = *(uchar*)(str+1) & ~0x80;
- c2 = *(uchar*)(str+2) & ~0x80;
- c3 = *(uchar*)(str+3) & ~0x80;
- n = (c<<18) | (c1<<12) | (c2<<6) | c3;
- if(n < 0x10000 || n > 0x10FFFF){
- *rune = Runeerror;
- return 1;
- }
- *rune = n;
- return 4;
-}
-
void
utfconv(void)
{
void
filetype(int fd)
{
- Rune1 r;
+ Rune r;
int i, f, n;
char *p, *eob;
language[i].count = 0;
eob = (char *)buf+nbuf;
for(n = 0, p = (char *)buf; p < eob; n++) {
- if (!fullrune1(p, eob-p) && eob-p < UTFmax1)
+ if (!fullrune(p, eob-p) && eob-p < UTFmax)
break;
- p += chartorune1(&r, p);
+ p += chartorune(&r, p);
if (r == 0)
f = Cnull;
else if (r <= 0x7f) {
#include <libc.h>
#include <bio.h>
-uvlong count[1<<16];
+uvlong count[Runemax+1];
Biobuf bout;
void usage(void);
x = re2or(x, rclass(ov, p[0]-1));
ov = p[1]+1;
}
- x = re2or(x, rclass(ov, 0xffff));
+ x = re2or(x, rclass(ov, Runemask));
} else {
x = rclass(p[0], p[1]);
for(p+=2; *p; p+=2)
Caselim = 7,
Nhunk = 1<<16,
- Cbegin = 0x10000,
+ Cbegin = Runemax+1,
Flshcnt = (1<<9)-1,
Cflag = 1<<0,
if(r == '\n')
return L("\n");
+ if(((uint)r&~0xFFFF) != 0){
+ /* The cache must grow a lot to handle them */
+ fprint(2, "%s: can't handle rune '%C'\n", argv0, r);
+ return L("?");
+ }
+
if(tcscache[r>>8] && tcscache[r>>8][r&0xFF])
return tcscache[r>>8][r&0xFF];
typedef struct Trtab Trtab;
struct Trtab
{
- char t[3];
+ char t[UTFmax];
Rune r;
};
int
equtf(uchar *p, uchar *q)
{
+ Rune pr, qr;
+
if(*p!=*q)
- return 0;
- if(twobyte(*p)) return p[1]==q[1];
- if(threebyte(*p)){
- if(p[1]!=q[1])
- return 0;
- if(p[1]=='\0')
- return 1; /* broken code at end of string! */
- return p[2]==q[2];
- }
- return 1;
+ return 0;
+
+ chartorune(&pr, (char*)p);
+ chartorune(&qr, (char*)q);
+ return pr == qr;
}
+
/*
* Return a pointer to the next utf code in the string,
* not jumping past nuls in broken utf codes!
uchar*
nextutf(uchar *p)
{
- if(twobyte(*p)) return p[1]=='\0'?p+1:p+2;
- if(threebyte(*p)) return p[1]=='\0'?p+1:p[2]=='\0'?p+2:p+3;
- return p+1;
+ Rune dummy;
+
+ return p + chartorune(&dummy, (char*)p);
}
+
/*
* Convert the utf code at *p to a unicode value
*/
int
unicode(uchar *p)
{
- int u = *p;
+ Rune r;
- if(twobyte(u))
- return ((u&0x1f)<<6)|(p[1]&0x3f);
- if(threebyte(u))
- return (u<<12)|((p[1]&0x3f)<<6)|(p[2]&0x3f);
- return u;
+ chartorune(&r, (char*)p);
+ return r;
}
+
/*
* Does the string s match the pattern p
* . and .. are only matched by patterns starting with .
char*
addutf(char *p, int c)
{
- p = addtok(p, c);
- if(twobyte(c)) /* 2-byte escape */
- return addtok(p, advance());
- if(threebyte(c)){ /* 3-byte escape */
+ uchar b, m;
+ int i;
+
+ p = addtok(p, c); /* 1-byte UTF runes are special */
+ if(onebyte(c))
+ return p;
+
+ m = 0xc0;
+ b = 0x80;
+ for(i=1; i < UTFmax; i++){
+ if((c&m) == b)
+ break;
p = addtok(p, advance());
- return addtok(p, advance());
+ b = m;
+ m = (m >> 1)|0x80;
}
return p;
}
+
int lastdol; /* was the last token read '$' or '$#' or '"'? */
int lastword; /* was the last token read a word or compound word terminator? */
*/
#define GLOB ((char)0x01)
/*
- * onebyte(c), twobyte(c), threebyte(c)
- * Is c the first character of a one- two- or three-byte utf sequence?
+ * onebyte(c)
+ * Is c the first character of a one-byte utf sequence?
*/
#define onebyte(c) ((c&0x80)==0x00)
-#define twobyte(c) ((c&0xe0)==0xc0)
-#define threebyte(c) ((c&0xf0)==0xe0)
char **argp;
char **args;
struct Inst
{
- long type; /* < 0x10000 ==> literal, otherwise action */
+ long type; /* <= Runemax ==> literal, otherwise action */
union {
int rsid;
int rsubid;
#define NLIST 127
-Ilist *tl, *nl; /* This list, next list */
+Ilist *tl, *nl; /* This list, next list */
Ilist list[2][NLIST+1]; /* +1 for trailing null */
static Rangeset sempty;
* 0x100xx are operators, value == precedence
* 0x200xx are tokens, i.e. operands for operators
*/
-#define OPERATOR 0x10000 /* Bitmask of all operators */
-#define START 0x10000 /* Start, used for marker on stack */
-#define RBRA 0x10001 /* Right bracket, ) */
-#define LBRA 0x10002 /* Left bracket, ( */
-#define OR 0x10003 /* Alternation, | */
-#define CAT 0x10004 /* Concatentation, implicit operator */
-#define STAR 0x10005 /* Closure, * */
-#define PLUS 0x10006 /* a+ == aa* */
-#define QUEST 0x10007 /* a? == a|nothing, i.e. 0 or 1 a's */
-#define ANY 0x20000 /* Any character but newline, . */
-#define NOP 0x20001 /* No operation, internal use only */
-#define BOL 0x20002 /* Beginning of line, ^ */
-#define EOL 0x20003 /* End of line, $ */
-#define CCLASS 0x20004 /* Character class, [] */
-#define NCCLASS 0x20005 /* Negated character class, [^] */
-#define END 0x20077 /* Terminate: match found */
-
-#define ISATOR 0x10000
-#define ISAND 0x20000
+enum {
+ OPERATOR = Runemask+1, /* Bitmask of all operators */
+ START = OPERATOR, /* Start, used for marker on stack */
+ RBRA, /* Right bracket, ) */
+ LBRA, /* Left bracket, ( */
+ OR, /* Alternation, | */
+ CAT, /* Concatentation, implicit operator */
+ STAR, /* Closure, * */
+ PLUS, /* a+ == aa* */
+ QUEST, /* a? == a|nothing, i.e. 0 or 1 a's */
+
+ ANY = OPERATOR<<1, /* Any character but newline, . */
+ NOP, /* No operation, internal use only */
+ BOL, /* Beginning of line, ^ */
+ EOL, /* End of line, $ */
+ CCLASS, /* Character class, [] */
+ NCCLASS, /* Negated character class, [^] */
+ END, /* Terminate: match found */
+
+ ISATOR = OPERATOR,
+ ISAND = OPERATOR<<1,
+};
/*
* Parser Information
exprp++;
return '\n';
}
- return *exprp++|0x10000;
+ return *exprp++|(Runemax+1);
}
return *exprp++;
}
#define CLEARBIT(a,c) ((a)[(c)/8] &= ~bits[(c)&07])
#define BITSET(a,c) ((a)[(c)/8] & bits[(c)&07])
-#define MAXRUNE Runemax
-
-uchar f[(MAXRUNE+1)/8];
-uchar t[(MAXRUNE+1)/8];
+uchar f[(Runemax+1)/8];
+uchar t[(Runemax+1)/8];
char wbuf[4096];
char *wptr;
projects / plan9front.git / commitdiff