Simpilfy mbe parsing

diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs
index 987a4f676d3eb2445f0f3cc7aac2292249bfa7e0..800931cd1d6a31040426457b4db9ca85179958f2 100644 (file)
--- a/crates/mbe/src/expander/matcher.rs
+++ b/crates/mbe/src/expander/matcher.rs
@@ -70,7 +70,7 @@ pub(super) struct Match {
 }
 
 impl Match {
-    pub(super) fn add_err(&mut self, err: ExpandError) {
+    fn add_err(&mut self, err: ExpandError) {
         let prev_err = self.err.take();
         self.err = prev_err.or(Some(err));
         self.err_count += 1;
@@ -79,12 +79,10 @@ pub(super) fn add_err(&mut self, err: ExpandError) {
 
 /// Matching errors are added to the `Match`.
 pub(super) fn match_(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
-    assert!(pattern.delimiter == None);
-
     let mut res = Match::default();
     let mut src = TtIter::new(src);
 
-    match_subtree(&mut res, pattern, &mut src);
+    match_tokens(&mut res, pattern, &mut src);
 
     if src.len() > 0 {
         res.unmatched_tts += src.len();
@@ -94,36 +92,16 @@ pub(super) fn match_(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
     res
 }
 
-fn match_subtree(res: &mut Match, pattern: &MetaTemplate, src: &mut TtIter) {
+fn match_tokens(res: &mut Match, pattern: &MetaTemplate, src: &mut TtIter) {
     for op in pattern.iter() {
         match op {
             Op::Leaf(lhs) => {
-                let rhs = match src.expect_leaf() {
-                    Ok(l) => l,
-                    Err(()) => {
-                        res.add_err(err!("expected leaf: `{}`", lhs));
-                        continue;
-                    }
-                };
-                match (lhs, rhs) {
-                    (
-                        tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
-                        tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
-                    ) if lhs == rhs => (),
-                    (
-                        tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
-                        tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
-                    ) if lhs == rhs => (),
-                    (
-                        tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
-                        tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
-                    ) if lhs == rhs => (),
-                    _ => {
-                        res.add_err(ExpandError::UnexpectedToken);
-                    }
+                if let Err(err) = match_leaf(lhs, src) {
+                    res.add_err(err);
+                    continue;
                 }
             }
-            Op::Subtree(lhs) => {
+            Op::Subtree { tokens, delimiter: delim } => {
                 let rhs = match src.expect_subtree() {
                     Ok(s) => s,
                     Err(()) => {
@@ -131,12 +109,12 @@ fn match_subtree(res: &mut Match, pattern: &MetaTemplate, src: &mut TtIter) {
                         continue;
                     }
                 };
-                if lhs.delimiter_kind() != rhs.delimiter_kind() {
+                if delim.map(|it| it.kind) != rhs.delimiter_kind() {
                     res.add_err(err!("mismatched delimiter"));
                     continue;
                 }
                 let mut src = TtIter::new(rhs);
-                match_subtree(res, lhs, &mut src);
+                match_tokens(res, tokens, &mut src);
                 if src.len() > 0 {
                     res.add_err(err!("leftover tokens"));
                 }
@@ -162,14 +140,42 @@ fn match_subtree(res: &mut Match, pattern: &MetaTemplate, src: &mut TtIter) {
                     res.add_err(err);
                 }
             }
-            Op::Repeat { subtree, kind, separator } => {
+            Op::Repeat { tokens: subtree, kind, separator } => {
                 match_repeat(res, subtree, *kind, separator, src);
             }
         }
     }
 }
 
-pub(super) fn match_repeat(
+fn match_leaf(lhs: &tt::Leaf, src: &mut TtIter) -> Result<(), ExpandError> {
+    let rhs = match src.expect_leaf() {
+        Ok(l) => l,
+        Err(()) => {
+            return Err(err!("expected leaf: `{}`", lhs));
+        }
+    };
+    match (lhs, rhs) {
+        (
+            tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
+            tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
+        ) if lhs == rhs => (),
+        (
+            tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
+            tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
+        ) if lhs == rhs => (),
+        (
+            tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
+            tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
+        ) if lhs == rhs => (),
+        _ => {
+            return Err(ExpandError::UnexpectedToken);
+        }
+    }
+
+    Ok(())
+}
+
+fn match_repeat(
     res: &mut Match,
     pattern: &MetaTemplate,
     kind: RepeatKind,
@@ -191,7 +197,7 @@ pub(super) fn match_repeat(
         }
 
         let mut nested = Match::default();
-        match_subtree(&mut nested, pattern, &mut fork);
+        match_tokens(&mut nested, pattern, &mut fork);
         if nested.err.is_none() {
             limit -= 1;
             if limit == 0 {
@@ -292,8 +298,8 @@ fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &MetaTemplate) {
         match op {
             Op::Var { name, .. } => buf.push(name.clone()),
             Op::Leaf(_) => (),
-            Op::Subtree(subtree) => collect_vars(buf, subtree),
-            Op::Repeat { subtree, .. } => collect_vars(buf, subtree),
+            Op::Subtree { tokens, .. } => collect_vars(buf, tokens),
+            Op::Repeat { tokens, .. } => collect_vars(buf, tokens),
         }
     }
 }
@@ -325,7 +331,7 @@ fn eat_separator(&mut self, separator: &Separator) -> bool {
         ok
     }
 
-    pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
+    fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
         match self.peek_n(0) {
             Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
                 return self.expect_lifetime();
@@ -386,7 +392,7 @@ pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
         }
     }
 
-    pub(crate) fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
+    fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
         let punct = self.expect_punct()?;
         if punct.char != '\'' {
             return Err(());
@@ -403,13 +409,13 @@ pub(crate) fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
         .into())
     }
 
-    pub(crate) fn expect_fragment(
+    fn expect_fragment(
         &mut self,
         fragment_kind: parser::FragmentKind,
     ) -> ExpandResult<Option<tt::TokenTree>> {
-        pub(crate) struct OffsetTokenSink<'a> {
-            pub(crate) cursor: Cursor<'a>,
-            pub(crate) error: bool,
+        struct OffsetTokenSink<'a> {
+            cursor: Cursor<'a>,
+            error: bool,
         }
 
         impl<'a> TreeSink for OffsetTokenSink<'a> {
@@ -465,7 +471,7 @@ fn error(&mut self, _error: parser::ParseError) {
         ExpandResult { value: res, err }
     }
 
-    pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> {
+    fn eat_vis(&mut self) -> Option<tt::TokenTree> {
         let mut fork = self.clone();
         match fork.expect_fragment(Visibility) {
             ExpandResult { value: tt, err: None } => {
@@ -476,7 +482,7 @@ pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> {
         }
     }
 
-    pub(crate) fn eat_char(&mut self, c: char) -> Option<tt::TokenTree> {
+    fn eat_char(&mut self, c: char) -> Option<tt::TokenTree> {
         let mut fork = self.clone();
         match fork.expect_char(c) {
             Ok(_) => {

diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs
index 30c090f32bde98b8fdb8a4fc19f3511d2c9a0918..78368a33e504cc1d7bd62460829fb3be062827b0 100644 (file)
--- a/crates/mbe/src/expander/transcriber.rs
+++ b/crates/mbe/src/expander/transcriber.rs
@@ -2,6 +2,7 @@
 //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
 
 use syntax::SmolStr;
+use tt::Delimiter;
 
 use super::ExpandResult;
 use crate::{
@@ -54,10 +55,9 @@ pub(super) fn transcribe(
     template: &MetaTemplate,
     bindings: &Bindings,
 ) -> ExpandResult<tt::Subtree> {
-    assert!(template.delimiter == None);
     let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() };
     let mut arena: Vec<tt::TokenTree> = Vec::new();
-    expand_subtree(&mut ctx, template, &mut arena)
+    expand_subtree(&mut ctx, template, None, &mut arena)
 }
 
 #[derive(Debug)]
@@ -80,6 +80,7 @@ struct ExpandCtx<'a> {
 fn expand_subtree(
     ctx: &mut ExpandCtx,
     template: &MetaTemplate,
+    delimiter: Option<Delimiter>,
     arena: &mut Vec<tt::TokenTree>,
 ) -> ExpandResult<tt::Subtree> {
     // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation
@@ -88,8 +89,9 @@ fn expand_subtree(
     for op in template.iter() {
         match op {
             Op::Leaf(tt) => arena.push(tt.clone().into()),
-            Op::Subtree(tt) => {
-                let ExpandResult { value: tt, err: e } = expand_subtree(ctx, &tt, arena);
+            Op::Subtree { tokens, delimiter } => {
+                let ExpandResult { value: tt, err: e } =
+                    expand_subtree(ctx, &tokens, *delimiter, arena);
                 err = err.or(e);
                 arena.push(tt.into());
             }
@@ -98,7 +100,7 @@ fn expand_subtree(
                 err = err.or(e);
                 push_fragment(arena, fragment);
             }
-            Op::Repeat { subtree, kind, separator } => {
+            Op::Repeat { tokens: subtree, kind, separator } => {
                 let ExpandResult { value: fragment, err: e } =
                     expand_repeat(ctx, subtree, *kind, separator, arena);
                 err = err.or(e);
@@ -108,7 +110,7 @@ fn expand_subtree(
     }
     // drain the elements added in this instance of expand_subtree
     let tts = arena.drain(start_elements..arena.len()).collect();
-    ExpandResult { value: tt::Subtree { delimiter: template.delimiter, token_trees: tts }, err }
+    ExpandResult { value: tt::Subtree { delimiter, token_trees: tts }, err }
 }
 
 fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> {
@@ -162,7 +164,7 @@ fn expand_repeat(
     let mut counter = 0;
 
     loop {
-        let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, arena);
+        let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, None, arena);
         let nesting_state = ctx.nesting.last_mut().unwrap();
         if nesting_state.at_end || !nesting_state.hit {
             break;

diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs
index 56c632665ea4e0beca1b0f60bc9efe82f40fd2da..d80bd7a3355f32203af70c6c56d1298d5fc42d6d 100644 (file)
--- a/crates/mbe/src/lib.rs
+++ b/crates/mbe/src/lib.rs
@@ -92,18 +92,11 @@ struct Rule {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
-struct MetaTemplate {
-    delimiter: Option<Delimiter>,
-    tokens: Vec<Op>,
-}
+struct MetaTemplate(Vec<Op>);
 
 impl<'a> MetaTemplate {
     fn iter(&self) -> impl Iterator<Item = &Op> {
-        self.tokens.iter()
-    }
-
-    fn delimiter_kind(&self) -> Option<DelimiterKind> {
-        self.delimiter.map(|it| it.kind)
+        self.0.iter()
     }
 }
 
@@ -288,8 +281,8 @@ fn parse(src: &mut TtIter, expect_arrow: bool) -> Result<Rule, ParseError> {
             .expect_subtree()
             .map_err(|()| ParseError::Expected("expected subtree".to_string()))?;
 
-        let lhs = MetaTemplate { tokens: parse_pattern(&lhs)?, delimiter: None };
-        let rhs = MetaTemplate { tokens: parse_template(&rhs)?, delimiter: None };
+        let lhs = MetaTemplate(parse_pattern(&lhs)?);
+        let rhs = MetaTemplate(parse_template(&rhs)?);
 
         Ok(crate::Rule { lhs, rhs })
     }
@@ -298,8 +291,8 @@ fn parse(src: &mut TtIter, expect_arrow: bool) -> Result<Rule, ParseError> {
 fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> {
     for op in pattern.iter() {
         match op {
-            Op::Subtree(subtree) => validate(&subtree)?,
-            Op::Repeat { subtree, separator, .. } => {
+            Op::Subtree { tokens, .. } => validate(&tokens)?,
+            Op::Repeat { tokens: subtree, separator, .. } => {
                 // Checks that no repetition which could match an empty token
                 // https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558
 
@@ -319,7 +312,7 @@ fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> {
                                 )
                             }
                             Op::Leaf(_) => {}
-                            Op::Subtree(_) => {}
+                            Op::Subtree { .. } => {}
                         }
                         false
                     }) {

diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs
index b90ae7015db7c811a5e97bed9165855cd14f4035..f891ec29c6e80b1dacf5fb88b6aac9a74de23da2 100644 (file)
--- a/crates/mbe/src/parser.rs
+++ b/crates/mbe/src/parser.rs
@@ -3,15 +3,16 @@
 
 use smallvec::SmallVec;
 use syntax::SmolStr;
+use tt::Delimiter;
 
 use crate::{tt_iter::TtIter, MetaTemplate, ParseError};
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(crate) enum Op {
     Var { name: SmolStr, kind: Option<SmolStr>, id: tt::TokenId },
-    Repeat { subtree: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
+    Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
     Leaf(tt::Leaf),
-    Subtree(MetaTemplate),
+    Subtree { tokens: MetaTemplate, delimiter: Option<Delimiter> },
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -92,12 +93,10 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
             match second {
                 tt::TokenTree::Subtree(subtree) => {
                     let (separator, kind) = parse_repeat(src)?;
-                    let delimiter = subtree.delimiter;
                     let tokens = parse_inner(&subtree, mode)
                         .into_iter()
                         .collect::<Result<Vec<Op>, ParseError>>()?;
-                    let subtree = MetaTemplate { tokens, delimiter };
-                    Op::Repeat { subtree, separator, kind }
+                    Op::Repeat { tokens: MetaTemplate(tokens), separator, kind }
                 }
                 tt::TokenTree::Leaf(leaf) => match leaf {
                     tt::Leaf::Punct(punct) => {
@@ -136,12 +135,9 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
         }
         tt::TokenTree::Leaf(tt) => Op::Leaf(tt.clone()),
         tt::TokenTree::Subtree(subtree) => {
-            let delimiter = subtree.delimiter;
             let tokens =
                 parse_inner(&subtree, mode).into_iter().collect::<Result<Vec<Op>, ParseError>>()?;
-
-            let subtree = MetaTemplate { tokens, delimiter };
-            Op::Subtree(subtree)
+            Op::Subtree { tokens: MetaTemplate(tokens), delimiter: subtree.delimiter }
         }
     };
     Ok(res)