if child == first_child || child == last_child || child.kind().is_trivia() {
continue;
}
- let child = if child.kind() == TOKEN_TREE {
+ let child: tt::TokenTree = if child.kind() == TOKEN_TREE {
convert_tt(child)?.into()
- } else if child.kind().is_keyword() {
+ } else if child.kind().is_keyword() || child.kind() == IDENT {
let text = child.leaf_text().unwrap().clone();
tt::Leaf::from(tt::Ident { text }).into()
+ } else if child.kind().is_punct() {
+ // FIXME: multibyte tokens
+ tt::Leaf::from(tt::Punct {
+ char: child.text().char_at(0)?,
+ })
+ .into()
+ } else if child.kind().is_literal() {
+ tt::Leaf::from(tt::Literal {
+ text: child.leaf_text().unwrap().clone(),
+ })
+ .into()
} else {
+ log::error!("unknown kind: {:?}", child);
return None;
};
token_trees.push(child)
SLASH,
CARET,
PERCENT,
+ UNDERSCORE,
DOT,
DOTDOT,
DOTDOTDOT,
AUTO_KW,
DEFAULT_KW,
UNION_KW,
- ERROR,
- IDENT,
- UNDERSCORE,
- WHITESPACE,
INT_NUMBER,
FLOAT_NUMBER,
- LIFETIME,
CHAR,
BYTE,
STRING,
RAW_STRING,
BYTE_STRING,
RAW_BYTE_STRING,
+ ERROR,
+ IDENT,
+ WHITESPACE,
+ LIFETIME,
COMMENT,
SHEBANG,
SOURCE_FILE,
}
}
+ pub fn is_punct(self) -> bool {
+ match self {
+ | SEMI
+ | COMMA
+ | L_PAREN
+ | R_PAREN
+ | L_CURLY
+ | R_CURLY
+ | L_BRACK
+ | R_BRACK
+ | L_ANGLE
+ | R_ANGLE
+ | AT
+ | POUND
+ | TILDE
+ | QUESTION
+ | DOLLAR
+ | AMP
+ | PIPE
+ | PLUS
+ | STAR
+ | SLASH
+ | CARET
+ | PERCENT
+ | UNDERSCORE
+ | DOT
+ | DOTDOT
+ | DOTDOTDOT
+ | DOTDOTEQ
+ | COLON
+ | COLONCOLON
+ | EQ
+ | EQEQ
+ | FAT_ARROW
+ | EXCL
+ | NEQ
+ | MINUS
+ | THIN_ARROW
+ | LTEQ
+ | GTEQ
+ | PLUSEQ
+ | MINUSEQ
+ | PIPEEQ
+ | AMPEQ
+ | CARETEQ
+ | SLASHEQ
+ | STAREQ
+ | PERCENTEQ
+ | AMPAMP
+ | PIPEPIPE
+ | SHL
+ | SHR
+ | SHLEQ
+ | SHREQ
+ => true,
+ _ => false
+ }
+ }
+ pub fn is_literal(self) -> bool {
+ match self {
+ | INT_NUMBER
+ | FLOAT_NUMBER
+ | CHAR
+ | BYTE
+ | STRING
+ | RAW_STRING
+ | BYTE_STRING
+ | RAW_BYTE_STRING
+ => true,
+ _ => false
+ }
+ }
+
pub(crate) fn info(self) -> &'static SyntaxInfo {
match self {
SEMI => &SyntaxInfo { name: "SEMI" },
SLASH => &SyntaxInfo { name: "SLASH" },
CARET => &SyntaxInfo { name: "CARET" },
PERCENT => &SyntaxInfo { name: "PERCENT" },
+ UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
DOT => &SyntaxInfo { name: "DOT" },
DOTDOT => &SyntaxInfo { name: "DOTDOT" },
DOTDOTDOT => &SyntaxInfo { name: "DOTDOTDOT" },
AUTO_KW => &SyntaxInfo { name: "AUTO_KW" },
DEFAULT_KW => &SyntaxInfo { name: "DEFAULT_KW" },
UNION_KW => &SyntaxInfo { name: "UNION_KW" },
- ERROR => &SyntaxInfo { name: "ERROR" },
- IDENT => &SyntaxInfo { name: "IDENT" },
- UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
- WHITESPACE => &SyntaxInfo { name: "WHITESPACE" },
INT_NUMBER => &SyntaxInfo { name: "INT_NUMBER" },
FLOAT_NUMBER => &SyntaxInfo { name: "FLOAT_NUMBER" },
- LIFETIME => &SyntaxInfo { name: "LIFETIME" },
CHAR => &SyntaxInfo { name: "CHAR" },
BYTE => &SyntaxInfo { name: "BYTE" },
STRING => &SyntaxInfo { name: "STRING" },
RAW_STRING => &SyntaxInfo { name: "RAW_STRING" },
BYTE_STRING => &SyntaxInfo { name: "BYTE_STRING" },
RAW_BYTE_STRING => &SyntaxInfo { name: "RAW_BYTE_STRING" },
+ ERROR => &SyntaxInfo { name: "ERROR" },
+ IDENT => &SyntaxInfo { name: "IDENT" },
+ WHITESPACE => &SyntaxInfo { name: "WHITESPACE" },
+ LIFETIME => &SyntaxInfo { name: "LIFETIME" },
COMMENT => &SyntaxInfo { name: "COMMENT" },
SHEBANG => &SyntaxInfo { name: "SHEBANG" },
SOURCE_FILE => &SyntaxInfo { name: "SOURCE_FILE" },
'/' => SLASH,
'^' => CARET,
'%' => PERCENT,
+ '_' => UNDERSCORE,
_ => return None,
};
Some(tok)
{% for kw in concat(a=keywords, b=contextual_keywords) %}
{{kw | upper}}_KW,
{%- endfor -%}
-{% for t in concat(a=tokens, b=nodes) %}
+{% for t in concat(a=literals, b=tokens, c=nodes) %}
{{t}},
{%- endfor %}
}
}
}
+ pub fn is_punct(self) -> bool {
+ match self {
+{%- for t in concat(a=single_byte_tokens, b=multi_byte_tokens) %}
+ | {{t.1}}
+{%- endfor %}
+ => true,
+ _ => false
+ }
+ }
+ pub fn is_literal(self) -> bool {
+ match self {
+{%- for t in literals %}
+ | {{t}}
+{%- endfor %}
+ => true,
+ _ => false
+ }
+ }
+
pub(crate) fn info(self) -> &'static SyntaxInfo {
match self {
{%- for t in concat(a=single_byte_tokens, b=multi_byte_tokens) %}
{% for kw in concat(a=keywords, b=contextual_keywords) %}
{{kw | upper}}_KW => &SyntaxInfo { name: "{{kw | upper}}_KW" },
{%- endfor -%}
-{% for t in concat(a=tokens, b=nodes) %}
+{% for t in concat(a=literals, b=tokens, c=nodes) %}
{{t}} => &SyntaxInfo { name: "{{t}}" },
{%- endfor %}
TOMBSTONE => &SyntaxInfo { name: "TOMBSTONE" },