From 5ba4f949c23dcf53f34995c90b7c01e6c641b1f0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 6 Nov 2020 22:21:56 +0100 Subject: [PATCH] Kill RAW_ literals Syntactically, they are indistinguishable from non-raw versions, so it doesn't make sense to separate then *at the syntax* level. --- crates/assists/src/assist_context.rs | 7 ++- crates/assists/src/handlers/add_turbo_fish.rs | 2 +- .../src/handlers/expand_glob_import.rs | 2 +- crates/assists/src/handlers/flip_comma.rs | 2 +- .../assists/src/handlers/flip_trait_bound.rs | 2 +- .../src/handlers/introduce_named_lifetime.rs | 2 +- crates/assists/src/handlers/invert_if.rs | 2 +- crates/assists/src/handlers/raw_string.rs | 31 +++++++---- crates/assists/src/handlers/remove_mut.rs | 2 +- .../src/handlers/replace_let_with_if_let.rs | 2 +- .../src/handlers/replace_string_with_char.rs | 2 +- crates/assists/src/handlers/split_import.rs | 2 +- crates/assists/src/handlers/unwrap_block.rs | 2 +- crates/ide/src/extend_selection.rs | 2 +- crates/ide/src/syntax_highlighting.rs | 16 +++--- crates/ide/src/syntax_highlighting/format.rs | 4 +- .../ide/src/syntax_highlighting/injection.rs | 2 +- crates/ide/src/syntax_tree.rs | 6 +-- crates/parser/src/grammar.rs | 5 +- crates/parser/src/grammar/expressions/atom.rs | 14 +---- crates/parser/src/grammar/items.rs | 4 +- crates/parser/src/syntax_kind/generated.rs | 5 +- crates/syntax/src/ast/expr_ext.rs | 11 +++- crates/syntax/src/ast/generated/tokens.rs | 8 +-- crates/syntax/src/ast/node_ext.rs | 8 +-- crates/syntax/src/ast/token_ext.rs | 52 ++++++++----------- crates/syntax/src/parsing/lexer.rs | 4 +- crates/syntax/src/parsing/reparsing.rs | 2 +- crates/syntax/src/validation.rs | 41 ++++++++------- .../err/0033_unclosed_raw_string_at_eof.txt | 2 +- .../0034_unclosed_raw_string_with_ferris.txt | 2 +- ..._unclosed_raw_string_with_ascii_escape.txt | 2 +- ...nclosed_raw_string_with_unicode_escape.txt | 2 +- .../0037_unclosed_raw_string_with_space.txt | 2 +- .../0038_unclosed_raw_string_with_slash.txt | 2 +- .../0039_unclosed_raw_string_with_slash_n.txt | 2 +- .../0040_unclosed_raw_byte_string_at_eof.txt | 2 +- ...1_unclosed_raw_byte_string_with_ferris.txt | 2 +- ...osed_raw_byte_string_with_ascii_escape.txt | 2 +- ...ed_raw_byte_string_with_unicode_escape.txt | 2 +- ...44_unclosed_raw_byte_string_with_space.txt | 2 +- ...45_unclosed_raw_byte_string_with_slash.txt | 2 +- ..._unclosed_raw_byte_string_with_slash_n.txt | 2 +- .../err/0047_unstarted_raw_string_at_eof.txt | 2 +- .../0048_unstarted_raw_byte_string_at_eof.txt | 2 +- .../0049_unstarted_raw_string_with_ascii.txt | 2 +- ...0_unstarted_raw_byte_string_with_ascii.txt | 2 +- .../test_data/lexer/ok/0008_byte_strings.txt | 4 +- .../test_data/lexer/ok/0009_strings.txt | 2 +- .../test_data/lexer/ok/0013_raw_strings.txt | 2 +- .../parser/inline/ok/0085_expr_literals.rast | 4 +- xtask/src/ast_src.rs | 11 +--- xtask/src/codegen/gen_syntax.rs | 2 +- 53 files changed, 141 insertions(+), 164 deletions(-) diff --git a/crates/assists/src/assist_context.rs b/crates/assists/src/assist_context.rs index d11fee196a3..fcfe2d6ee4a 100644 --- a/crates/assists/src/assist_context.rs +++ b/crates/assists/src/assist_context.rs @@ -12,7 +12,7 @@ }; use syntax::{ algo::{self, find_node_at_offset, SyntaxRewriter}, - AstNode, SourceFile, SyntaxElement, SyntaxKind, SyntaxToken, TextRange, TextSize, + AstNode, AstToken, SourceFile, SyntaxElement, SyntaxKind, SyntaxToken, TextRange, TextSize, TokenAtOffset, }; use text_edit::{TextEdit, TextEditBuilder}; @@ -81,9 +81,12 @@ pub(crate) fn offset(&self) -> TextSize { pub(crate) fn token_at_offset(&self) -> TokenAtOffset { self.source_file.syntax().token_at_offset(self.offset()) } - pub(crate) fn find_token_at_offset(&self, kind: SyntaxKind) -> Option { + pub(crate) fn find_token_syntax_at_offset(&self, kind: SyntaxKind) -> Option { self.token_at_offset().find(|it| it.kind() == kind) } + pub(crate) fn find_token_at_offset(&self) -> Option { + self.token_at_offset().find_map(T::cast) + } pub(crate) fn find_node_at_offset(&self) -> Option { find_node_at_offset(self.source_file.syntax(), self.offset()) } diff --git a/crates/assists/src/handlers/add_turbo_fish.rs b/crates/assists/src/handlers/add_turbo_fish.rs index e3d84d6986a..1f486c013c8 100644 --- a/crates/assists/src/handlers/add_turbo_fish.rs +++ b/crates/assists/src/handlers/add_turbo_fish.rs @@ -25,7 +25,7 @@ // } // ``` pub(crate) fn add_turbo_fish(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let ident = ctx.find_token_at_offset(SyntaxKind::IDENT).or_else(|| { + let ident = ctx.find_token_syntax_at_offset(SyntaxKind::IDENT).or_else(|| { let arg_list = ctx.find_node_at_offset::()?; if arg_list.args().count() > 0 { return None; diff --git a/crates/assists/src/handlers/expand_glob_import.rs b/crates/assists/src/handlers/expand_glob_import.rs index 316a58d883a..853266395da 100644 --- a/crates/assists/src/handlers/expand_glob_import.rs +++ b/crates/assists/src/handlers/expand_glob_import.rs @@ -41,7 +41,7 @@ // fn qux(bar: Bar, baz: Baz) {} // ``` pub(crate) fn expand_glob_import(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let star = ctx.find_token_at_offset(T![*])?; + let star = ctx.find_token_syntax_at_offset(T![*])?; let (parent, mod_path) = find_parent_and_path(&star)?; let target_module = match ctx.sema.resolve_path(&mod_path)? { PathResolution::Def(ModuleDef::Module(it)) => it, diff --git a/crates/assists/src/handlers/flip_comma.rs b/crates/assists/src/handlers/flip_comma.rs index 5c69db53e56..64b4b1a76a5 100644 --- a/crates/assists/src/handlers/flip_comma.rs +++ b/crates/assists/src/handlers/flip_comma.rs @@ -18,7 +18,7 @@ // } // ``` pub(crate) fn flip_comma(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let comma = ctx.find_token_at_offset(T![,])?; + let comma = ctx.find_token_syntax_at_offset(T![,])?; let prev = non_trivia_sibling(comma.clone().into(), Direction::Prev)?; let next = non_trivia_sibling(comma.clone().into(), Direction::Next)?; diff --git a/crates/assists/src/handlers/flip_trait_bound.rs b/crates/assists/src/handlers/flip_trait_bound.rs index 347e79b1dad..92ee4218104 100644 --- a/crates/assists/src/handlers/flip_trait_bound.rs +++ b/crates/assists/src/handlers/flip_trait_bound.rs @@ -20,7 +20,7 @@ pub(crate) fn flip_trait_bound(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { // We want to replicate the behavior of `flip_binexpr` by only suggesting // the assist when the cursor is on a `+` - let plus = ctx.find_token_at_offset(T![+])?; + let plus = ctx.find_token_syntax_at_offset(T![+])?; // Make sure we're in a `TypeBoundList` if ast::TypeBoundList::cast(plus.parent()).is_none() { diff --git a/crates/assists/src/handlers/introduce_named_lifetime.rs b/crates/assists/src/handlers/introduce_named_lifetime.rs index 5f623e5f7ad..4cc8dae6502 100644 --- a/crates/assists/src/handlers/introduce_named_lifetime.rs +++ b/crates/assists/src/handlers/introduce_named_lifetime.rs @@ -36,7 +36,7 @@ // FIXME: should also add support for the case fun(f: &Foo) -> &<|>Foo pub(crate) fn introduce_named_lifetime(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { let lifetime_token = ctx - .find_token_at_offset(SyntaxKind::LIFETIME) + .find_token_syntax_at_offset(SyntaxKind::LIFETIME) .filter(|lifetime| lifetime.text() == "'_")?; if let Some(fn_def) = lifetime_token.ancestors().find_map(ast::Fn::cast) { generate_fn_def_assist(acc, &fn_def, lifetime_token.text_range()) diff --git a/crates/assists/src/handlers/invert_if.rs b/crates/assists/src/handlers/invert_if.rs index 461fcf8626b..ea722b91b2d 100644 --- a/crates/assists/src/handlers/invert_if.rs +++ b/crates/assists/src/handlers/invert_if.rs @@ -29,7 +29,7 @@ // ``` pub(crate) fn invert_if(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let if_keyword = ctx.find_token_at_offset(T![if])?; + let if_keyword = ctx.find_token_syntax_at_offset(T![if])?; let expr = ast::IfExpr::cast(if_keyword.parent())?; let if_range = if_keyword.text_range(); let cursor_in_range = if_range.contains_range(ctx.frange.range); diff --git a/crates/assists/src/handlers/raw_string.rs b/crates/assists/src/handlers/raw_string.rs index 9ddd116e015..7f9f01c9cd2 100644 --- a/crates/assists/src/handlers/raw_string.rs +++ b/crates/assists/src/handlers/raw_string.rs @@ -2,9 +2,7 @@ use syntax::{ ast::{self, HasQuotes, HasStringValue}, - AstToken, - SyntaxKind::{RAW_STRING, STRING}, - TextRange, TextSize, + AstToken, TextRange, TextSize, }; use test_utils::mark; @@ -26,7 +24,10 @@ // } // ``` pub(crate) fn make_raw_string(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let token = ctx.find_token_at_offset(STRING).and_then(ast::String::cast)?; + let token = ctx.find_token_at_offset::()?; + if token.is_raw() { + return None; + } let value = token.value()?; let target = token.syntax().text_range(); acc.add( @@ -65,7 +66,10 @@ pub(crate) fn make_raw_string(acc: &mut Assists, ctx: &AssistContext) -> Option< // } // ``` pub(crate) fn make_usual_string(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let token = ctx.find_token_at_offset(RAW_STRING).and_then(ast::RawString::cast)?; + let token = ctx.find_token_at_offset::()?; + if !token.is_raw() { + return None; + } let value = token.value()?; let target = token.syntax().text_range(); acc.add( @@ -104,11 +108,15 @@ pub(crate) fn make_usual_string(acc: &mut Assists, ctx: &AssistContext) -> Optio // } // ``` pub(crate) fn add_hash(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let token = ctx.find_token_at_offset(RAW_STRING)?; - let target = token.text_range(); + let token = ctx.find_token_at_offset::()?; + if !token.is_raw() { + return None; + } + let text_range = token.syntax().text_range(); + let target = text_range; acc.add(AssistId("add_hash", AssistKind::Refactor), "Add #", target, |edit| { - edit.insert(token.text_range().start() + TextSize::of('r'), "#"); - edit.insert(token.text_range().end(), "#"); + edit.insert(text_range.start() + TextSize::of('r'), "#"); + edit.insert(text_range.end(), "#"); }) } @@ -128,7 +136,10 @@ pub(crate) fn add_hash(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { // } // ``` pub(crate) fn remove_hash(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let token = ctx.find_token_at_offset(RAW_STRING).and_then(ast::RawString::cast)?; + let token = ctx.find_token_at_offset::()?; + if !token.is_raw() { + return None; + } let text = token.text().as_str(); if !text.starts_with("r#") && text.ends_with('#') { diff --git a/crates/assists/src/handlers/remove_mut.rs b/crates/assists/src/handlers/remove_mut.rs index 44f41daa92d..575b271f71d 100644 --- a/crates/assists/src/handlers/remove_mut.rs +++ b/crates/assists/src/handlers/remove_mut.rs @@ -18,7 +18,7 @@ // } // ``` pub(crate) fn remove_mut(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let mut_token = ctx.find_token_at_offset(T![mut])?; + let mut_token = ctx.find_token_syntax_at_offset(T![mut])?; let delete_from = mut_token.text_range().start(); let delete_to = match mut_token.next_token() { Some(it) if it.kind() == SyntaxKind::WHITESPACE => it.text_range().end(), diff --git a/crates/assists/src/handlers/replace_let_with_if_let.rs b/crates/assists/src/handlers/replace_let_with_if_let.rs index a5bcbda24e1..69d3b08d36c 100644 --- a/crates/assists/src/handlers/replace_let_with_if_let.rs +++ b/crates/assists/src/handlers/replace_let_with_if_let.rs @@ -37,7 +37,7 @@ // fn compute() -> Option { None } // ``` pub(crate) fn replace_let_with_if_let(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let let_kw = ctx.find_token_at_offset(T![let])?; + let let_kw = ctx.find_token_syntax_at_offset(T![let])?; let let_stmt = let_kw.ancestors().find_map(ast::LetStmt::cast)?; let init = let_stmt.initializer()?; let original_pat = let_stmt.pat()?; diff --git a/crates/assists/src/handlers/replace_string_with_char.rs b/crates/assists/src/handlers/replace_string_with_char.rs index 4ca87a8ecc0..6d227e8831d 100644 --- a/crates/assists/src/handlers/replace_string_with_char.rs +++ b/crates/assists/src/handlers/replace_string_with_char.rs @@ -22,7 +22,7 @@ // } // ``` pub(crate) fn replace_string_with_char(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let token = ctx.find_token_at_offset(STRING).and_then(ast::String::cast)?; + let token = ctx.find_token_syntax_at_offset(STRING).and_then(ast::String::cast)?; let value = token.value()?; let target = token.syntax().text_range(); diff --git a/crates/assists/src/handlers/split_import.rs b/crates/assists/src/handlers/split_import.rs index 15e67eaa19f..ef1f6b8a1aa 100644 --- a/crates/assists/src/handlers/split_import.rs +++ b/crates/assists/src/handlers/split_import.rs @@ -16,7 +16,7 @@ // use std::{collections::HashMap}; // ``` pub(crate) fn split_import(acc: &mut Assists, ctx: &AssistContext) -> Option<()> { - let colon_colon = ctx.find_token_at_offset(T![::])?; + let colon_colon = ctx.find_token_syntax_at_offset(T![::])?; let path = ast::Path::cast(colon_colon.parent())?.qualifier()?; let top_path = successors(Some(path.clone()), |it| it.parent_path()).last()?; diff --git a/crates/assists/src/handlers/unwrap_block.rs b/crates/assists/src/handlers/unwrap_block.rs index 3851aeb3e74..36ef871b938 100644 --- a/crates/assists/src/handlers/unwrap_block.rs +++ b/crates/assists/src/handlers/unwrap_block.rs @@ -29,7 +29,7 @@ pub(crate) fn unwrap_block(acc: &mut Assists, ctx: &AssistContext) -> Option<()> let assist_id = AssistId("unwrap_block", AssistKind::RefactorRewrite); let assist_label = "Unwrap block"; - let l_curly_token = ctx.find_token_at_offset(T!['{'])?; + let l_curly_token = ctx.find_token_syntax_at_offset(T!['{'])?; let mut block = ast::BlockExpr::cast(l_curly_token.parent())?; let mut parent = block.syntax().parent()?; if ast::MatchArm::can_cast(parent.kind()) { diff --git a/crates/ide/src/extend_selection.rs b/crates/ide/src/extend_selection.rs index 3ee0af8ad43..0971f770153 100644 --- a/crates/ide/src/extend_selection.rs +++ b/crates/ide/src/extend_selection.rs @@ -35,7 +35,7 @@ fn try_extend_selection( ) -> Option { let range = frange.range; - let string_kinds = [COMMENT, STRING, RAW_STRING, BYTE_STRING, RAW_BYTE_STRING]; + let string_kinds = [COMMENT, STRING, BYTE_STRING]; let list_kinds = [ RECORD_PAT_FIELD_LIST, MATCH_ARM_LIST, diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index efcc8ecfe6b..05bafe9c89d 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -179,10 +179,12 @@ pub(crate) fn highlight( element.clone() }; - if let Some(token) = element.as_token().cloned().and_then(ast::RawString::cast) { - let expanded = element_to_highlight.as_token().unwrap().clone(); - if injection::highlight_injection(&mut stack, &sema, token, expanded).is_some() { - continue; + if let Some(token) = element.as_token().cloned().and_then(ast::String::cast) { + if token.is_raw() { + let expanded = element_to_highlight.as_token().unwrap().clone(); + if injection::highlight_injection(&mut stack, &sema, token, expanded).is_some() { + continue; + } } } @@ -214,10 +216,6 @@ pub(crate) fn highlight( } stack.pop_and_inject(None); } - } else if let Some(string) = - element_to_highlight.as_token().cloned().and_then(ast::RawString::cast) - { - format_string_highlighter.highlight_format_string(&mut stack, &string, range); } } } @@ -532,7 +530,7 @@ fn highlight_element( None => h.into(), } } - STRING | RAW_STRING | RAW_BYTE_STRING | BYTE_STRING => HighlightTag::StringLiteral.into(), + STRING | BYTE_STRING => HighlightTag::StringLiteral.into(), ATTR => HighlightTag::Attribute.into(), INT_NUMBER | FLOAT_NUMBER => HighlightTag::NumericLiteral.into(), BYTE => HighlightTag::ByteLiteral.into(), diff --git a/crates/ide/src/syntax_highlighting/format.rs b/crates/ide/src/syntax_highlighting/format.rs index 71bde24f089..42f27df5d9b 100644 --- a/crates/ide/src/syntax_highlighting/format.rs +++ b/crates/ide/src/syntax_highlighting/format.rs @@ -29,9 +29,7 @@ pub(super) fn check_for_format_string(&mut self, parent: &SyntaxNode) { .children_with_tokens() .filter(|t| t.kind() != SyntaxKind::WHITESPACE) .nth(1) - .filter(|e| { - ast::String::can_cast(e.kind()) || ast::RawString::can_cast(e.kind()) - }) + .filter(|e| ast::String::can_cast(e.kind())) } _ => {} } diff --git a/crates/ide/src/syntax_highlighting/injection.rs b/crates/ide/src/syntax_highlighting/injection.rs index 59a74bc02e5..79f6b535991 100644 --- a/crates/ide/src/syntax_highlighting/injection.rs +++ b/crates/ide/src/syntax_highlighting/injection.rs @@ -15,7 +15,7 @@ pub(super) fn highlight_injection( acc: &mut HighlightedRangeStack, sema: &Semantics, - literal: ast::RawString, + literal: ast::String, expanded: SyntaxToken, ) -> Option<()> { let active_parameter = ActiveParameter::at_token(&sema, expanded)?; diff --git a/crates/ide/src/syntax_tree.rs b/crates/ide/src/syntax_tree.rs index 7941610d6b0..6dd05c05de4 100644 --- a/crates/ide/src/syntax_tree.rs +++ b/crates/ide/src/syntax_tree.rs @@ -1,9 +1,7 @@ use ide_db::base_db::{FileId, SourceDatabase}; use ide_db::RootDatabase; use syntax::{ - algo, AstNode, NodeOrToken, SourceFile, - SyntaxKind::{RAW_STRING, STRING}, - SyntaxToken, TextRange, TextSize, + algo, AstNode, NodeOrToken, SourceFile, SyntaxKind::STRING, SyntaxToken, TextRange, TextSize, }; // Feature: Show Syntax Tree @@ -46,7 +44,7 @@ fn syntax_tree_for_string(token: &SyntaxToken, text_range: TextRange) -> Option< // we'll attempt parsing it as rust syntax // to provide the syntax tree of the contents of the string match token.kind() { - STRING | RAW_STRING => syntax_tree_for_token(token, text_range), + STRING => syntax_tree_for_token(token, text_range), _ => None, } } diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 4ab206a838e..116b991a820 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -236,10 +236,7 @@ fn abi(p: &mut Parser) { assert!(p.at(T![extern])); let abi = p.start(); p.bump(T![extern]); - match p.current() { - STRING | RAW_STRING => p.bump_any(), - _ => (), - } + p.eat(STRING); abi.complete(p, ABI); } diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 66a92a4e111..31f42f16185 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -15,18 +15,8 @@ // let _ = b"e"; // let _ = br"f"; // } -pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[ - TRUE_KW, - FALSE_KW, - INT_NUMBER, - FLOAT_NUMBER, - BYTE, - CHAR, - STRING, - RAW_STRING, - BYTE_STRING, - RAW_BYTE_STRING, -]); +pub(crate) const LITERAL_FIRST: TokenSet = + TokenSet::new(&[TRUE_KW, FALSE_KW, INT_NUMBER, FLOAT_NUMBER, BYTE, CHAR, STRING, BYTE_STRING]); pub(crate) fn literal(p: &mut Parser) -> Option { if !p.at_ts(LITERAL_FIRST) { diff --git a/crates/parser/src/grammar/items.rs b/crates/parser/src/grammar/items.rs index 22810e6fb88..780bc470a43 100644 --- a/crates/parser/src/grammar/items.rs +++ b/crates/parser/src/grammar/items.rs @@ -239,9 +239,7 @@ fn items_without_modifiers(p: &mut Parser, m: Marker) -> Result<(), Marker> { T![static] => consts::static_(p, m), // test extern_block // extern {} - T![extern] - if la == T!['{'] || ((la == STRING || la == RAW_STRING) && p.nth(2) == T!['{']) => - { + T![extern] if la == T!['{'] || (la == STRING && p.nth(2) == T!['{']) => { abi(p); extern_item_list(p); m.complete(p, EXTERN_BLOCK); diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 935bd2c5e93..8bc6688f3cc 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -111,9 +111,7 @@ pub enum SyntaxKind { CHAR, BYTE, STRING, - RAW_STRING, BYTE_STRING, - RAW_BYTE_STRING, ERROR, IDENT, WHITESPACE, @@ -277,8 +275,7 @@ pub fn is_punct(self) -> bool { } pub fn is_literal(self) -> bool { match self { - INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | RAW_STRING | BYTE_STRING - | RAW_BYTE_STRING => true, + INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING => true, _ => false, } } diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs index 3d33cd1cf79..eb44bb2ab17 100644 --- a/crates/syntax/src/ast/expr_ext.rs +++ b/crates/syntax/src/ast/expr_ext.rs @@ -320,6 +320,13 @@ pub fn as_int_number(&self) -> Option { ast::IntNumber::cast(self.token()) } + pub fn as_string(&self) -> Option { + ast::String::cast(self.token()) + } + pub fn as_byte_string(&self) -> Option { + ast::ByteString::cast(self.token()) + } + fn find_suffix(text: &str, possible_suffixes: &[&str]) -> Option { possible_suffixes .iter() @@ -351,10 +358,10 @@ pub fn kind(&self) -> LiteralKind { suffix: Self::find_suffix(&text, &ast::FloatNumber::SUFFIXES), } } - STRING | RAW_STRING => LiteralKind::String, + STRING => LiteralKind::String, T![true] => LiteralKind::Bool(true), T![false] => LiteralKind::Bool(false), - BYTE_STRING | RAW_BYTE_STRING => LiteralKind::ByteString, + BYTE_STRING => LiteralKind::ByteString, CHAR => LiteralKind::Char, BYTE => LiteralKind::Byte, _ => unreachable!(), diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs index 1b8449221d3..728b72cd77e 100644 --- a/crates/syntax/src/ast/generated/tokens.rs +++ b/crates/syntax/src/ast/generated/tokens.rs @@ -70,16 +70,16 @@ fn syntax(&self) -> &SyntaxToken { &self.syntax } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct RawString { +pub struct ByteString { pub(crate) syntax: SyntaxToken, } -impl std::fmt::Display for RawString { +impl std::fmt::Display for ByteString { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.syntax, f) } } -impl AstToken for RawString { - fn can_cast(kind: SyntaxKind) -> bool { kind == RAW_STRING } +impl AstToken for ByteString { + fn can_cast(kind: SyntaxKind) -> bool { kind == BYTE_STRING } fn cast(syntax: SyntaxToken) -> Option { if Self::can_cast(syntax.kind()) { Some(Self { syntax }) diff --git a/crates/syntax/src/ast/node_ext.rs b/crates/syntax/src/ast/node_ext.rs index c5cd1c504da..5579f72b978 100644 --- a/crates/syntax/src/ast/node_ext.rs +++ b/crates/syntax/src/ast/node_ext.rs @@ -55,13 +55,7 @@ pub fn as_simple_key_value(&self) -> Option<(SmolStr, SmolStr)> { let key = self.simple_name()?; let value_token = lit.syntax().first_token()?; - let value: SmolStr = if let Some(s) = ast::String::cast(value_token.clone()) { - s.value()?.into() - } else if let Some(s) = ast::RawString::cast(value_token) { - s.value()?.into() - } else { - return None; - }; + let value: SmolStr = ast::String::cast(value_token.clone())?.value()?.into(); Some((key, value)) } diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index 8d3fad5a6c6..6cd20b6a607 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -139,14 +139,31 @@ fn text_range_between_quotes(&self) -> Option { } impl HasQuotes for ast::String {} -impl HasQuotes for ast::RawString {} pub trait HasStringValue: HasQuotes { fn value(&self) -> Option>; } +impl ast::String { + pub fn is_raw(&self) -> bool { + self.text().starts_with('r') + } + pub fn map_range_up(&self, range: TextRange) -> Option { + let contents_range = self.text_range_between_quotes()?; + assert!(TextRange::up_to(contents_range.len()).contains_range(range)); + Some(range + contents_range.start()) + } +} + impl HasStringValue for ast::String { fn value(&self) -> Option> { + if self.is_raw() { + let text = self.text().as_str(); + let text = + &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; + return Some(Cow::Borrowed(text)); + } + let text = self.text().as_str(); let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; @@ -166,20 +183,9 @@ fn value(&self) -> Option> { } } -// FIXME: merge `ast::RawString` and `ast::String`. -impl HasStringValue for ast::RawString { - fn value(&self) -> Option> { - let text = self.text().as_str(); - let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; - Some(Cow::Borrowed(text)) - } -} - -impl ast::RawString { - pub fn map_range_up(&self, range: TextRange) -> Option { - let contents_range = self.text_range_between_quotes()?; - assert!(TextRange::up_to(contents_range.len()).contains_range(range)); - Some(range + contents_range.start()) +impl ast::ByteString { + pub fn is_raw(&self) -> bool { + self.text().starts_with("br") } } @@ -522,22 +528,6 @@ fn char_ranges( } } -impl HasFormatSpecifier for ast::RawString { - fn char_ranges( - &self, - ) -> Option)>> { - let text = self.text().as_str(); - let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; - let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start(); - - let mut res = Vec::with_capacity(text.len()); - for (idx, c) in text.char_indices() { - res.push((TextRange::at(idx.try_into().unwrap(), TextSize::of(c)) + offset, Ok(c))); - } - Some(res) - } -} - impl ast::IntNumber { #[rustfmt::skip] pub(crate) const SUFFIXES: &'static [&'static str] = &[ diff --git a/crates/syntax/src/parsing/lexer.rs b/crates/syntax/src/parsing/lexer.rs index 5674ecb84db..8afd7e53bc5 100644 --- a/crates/syntax/src/parsing/lexer.rs +++ b/crates/syntax/src/parsing/lexer.rs @@ -235,7 +235,7 @@ fn match_literal_kind(kind: &rustc_lexer::LiteralKind) -> (SyntaxKind, Option<&' RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols", }; }; - RAW_STRING + STRING } rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => { if let Some(raw_str_err) = raw_str_err { @@ -250,7 +250,7 @@ fn match_literal_kind(kind: &rustc_lexer::LiteralKind) -> (SyntaxKind, Option<&' }; }; - RAW_BYTE_STRING + BYTE_STRING } }; diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs index 4149f856a83..190f5f67a45 100644 --- a/crates/syntax/src/parsing/reparsing.rs +++ b/crates/syntax/src/parsing/reparsing.rs @@ -44,7 +44,7 @@ fn reparse_token<'node>( let prev_token = algo::find_covering_element(root, edit.delete).as_token()?.clone(); let prev_token_kind = prev_token.kind(); match prev_token_kind { - WHITESPACE | COMMENT | IDENT | STRING | RAW_STRING => { + WHITESPACE | COMMENT | IDENT | STRING => { if prev_token_kind == WHITESPACE || prev_token_kind == COMMENT { // removing a new line may extends previous token let deleted_range = edit.delete - prev_token.text_range().start(); diff --git a/crates/syntax/src/validation.rs b/crates/syntax/src/validation.rs index 0f9a5e8ae7e..62a37c50a15 100644 --- a/crates/syntax/src/validation.rs +++ b/crates/syntax/src/validation.rs @@ -4,7 +4,7 @@ use crate::{ algo, ast, match_ast, AstNode, SyntaxError, - SyntaxKind::{BYTE, BYTE_STRING, CHAR, CONST, FN, INT_NUMBER, STRING, TYPE_ALIAS}, + SyntaxKind::{BYTE, CHAR, CONST, FN, INT_NUMBER, TYPE_ALIAS}, SyntaxNode, SyntaxToken, TextSize, T, }; use rowan::Direction; @@ -121,18 +121,19 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> { acc.push(SyntaxError::new_at_offset(rustc_unescape_error_to_string(err), off)); }; - match token.kind() { - BYTE => { - if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) { - push_err(2, e); - } - } - CHAR => { - if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) { - push_err(1, e); + if let Some(s) = literal.as_string() { + if !s.is_raw() { + if let Some(without_quotes) = unquote(text, 1, '"') { + unescape_literal(without_quotes, Mode::Str, &mut |range, char| { + if let Err(err) = char { + push_err(1, (range.start, err)); + } + }) } } - BYTE_STRING => { + } + if let Some(s) = literal.as_byte_string() { + if !s.is_raw() { if let Some(without_quotes) = unquote(text, 2, '"') { unescape_byte_literal(without_quotes, Mode::ByteStr, &mut |range, char| { if let Err(err) = char { @@ -141,13 +142,17 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> { }) } } - STRING => { - if let Some(without_quotes) = unquote(text, 1, '"') { - unescape_literal(without_quotes, Mode::Str, &mut |range, char| { - if let Err(err) = char { - push_err(1, (range.start, err)); - } - }) + } + + match token.kind() { + BYTE => { + if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) { + push_err(2, e); + } + } + CHAR => { + if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) { + push_err(1, e); } } _ => (), diff --git a/crates/syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt index 6fd59ccc027..54e707b7367 100644 --- a/crates/syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt +++ b/crates/syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt @@ -1,2 +1,2 @@ -RAW_STRING 4 "r##\"" +STRING 4 "r##\"" > error0..4 token("r##\"") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt b/crates/syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt index 8d9ca0e8f12..1f9889775f5 100644 --- a/crates/syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt +++ b/crates/syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt @@ -1,2 +1,2 @@ -RAW_STRING 8 "r##\"🦀" +STRING 8 "r##\"🦀" > error0..8 token("r##\"🦀") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt b/crates/syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt index a906380c718..93f6f72ae60 100644 --- a/crates/syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt +++ b/crates/syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt @@ -1,2 +1,2 @@ -RAW_STRING 8 "r##\"\\x7f" +STRING 8 "r##\"\\x7f" > error0..8 token("r##\"\\x7f") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt b/crates/syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt index 5667c614995..1d2ebc60fdf 100644 --- a/crates/syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt +++ b/crates/syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt @@ -1,2 +1,2 @@ -RAW_STRING 12 "r##\"\\u{20AA}" +STRING 12 "r##\"\\u{20AA}" > error0..12 token("r##\"\\u{20AA}") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt b/crates/syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt index 141c8268e80..c567ab7e282 100644 --- a/crates/syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt +++ b/crates/syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt @@ -1,2 +1,2 @@ -RAW_STRING 5 "r##\" " +STRING 5 "r##\" " > error0..5 token("r##\" ") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt b/crates/syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt index f61d4cc915e..343b20323a1 100644 --- a/crates/syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt +++ b/crates/syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt @@ -1,2 +1,2 @@ -RAW_STRING 5 "r##\"\\" +STRING 5 "r##\"\\" > error0..5 token("r##\"\\") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt b/crates/syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt index 12e2c0fc038..041a4273758 100644 --- a/crates/syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt +++ b/crates/syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt @@ -1,2 +1,2 @@ -RAW_STRING 6 "r##\"\\n" +STRING 6 "r##\"\\n" > error0..6 token("r##\"\\n") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt index fe12cb5fc6d..efaa1cafd93 100644 --- a/crates/syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt +++ b/crates/syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt @@ -1,2 +1,2 @@ -RAW_BYTE_STRING 5 "br##\"" +BYTE_STRING 5 "br##\"" > error0..5 token("br##\"") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt b/crates/syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt index 5be2a7861b4..b6c938f94e9 100644 --- a/crates/syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt +++ b/crates/syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt @@ -1,2 +1,2 @@ -RAW_BYTE_STRING 9 "br##\"🦀" +BYTE_STRING 9 "br##\"🦀" > error0..9 token("br##\"🦀") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt b/crates/syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt index 6cbe08d07ed..f82efe49acc 100644 --- a/crates/syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt +++ b/crates/syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt @@ -1,2 +1,2 @@ -RAW_BYTE_STRING 9 "br##\"\\x7f" +BYTE_STRING 9 "br##\"\\x7f" > error0..9 token("br##\"\\x7f") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt b/crates/syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt index f56a4f98480..4e4a576961a 100644 --- a/crates/syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt +++ b/crates/syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt @@ -1,2 +1,2 @@ -RAW_BYTE_STRING 13 "br##\"\\u{20AA}" +BYTE_STRING 13 "br##\"\\u{20AA}" > error0..13 token("br##\"\\u{20AA}") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt b/crates/syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt index 3d32ce34ea5..0018c8623c2 100644 --- a/crates/syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt +++ b/crates/syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt @@ -1,2 +1,2 @@ -RAW_BYTE_STRING 6 "br##\" " +BYTE_STRING 6 "br##\" " > error0..6 token("br##\" ") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt b/crates/syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt index 320fea17700..c3ba4ae82b8 100644 --- a/crates/syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt +++ b/crates/syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt @@ -1,2 +1,2 @@ -RAW_BYTE_STRING 6 "br##\"\\" +BYTE_STRING 6 "br##\"\\" > error0..6 token("br##\"\\") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt b/crates/syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt index b3a56380cbd..7bda72276fe 100644 --- a/crates/syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt +++ b/crates/syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt @@ -1,2 +1,2 @@ -RAW_BYTE_STRING 7 "br##\"\\n" +BYTE_STRING 7 "br##\"\\n" > error0..7 token("br##\"\\n") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt index 5af1e2d97ff..ce92d2ff75b 100644 --- a/crates/syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt +++ b/crates/syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt @@ -1,2 +1,2 @@ -RAW_STRING 3 "r##" +STRING 3 "r##" > error0..3 token("r##") msg(Missing `"` symbol after `#` symbols to begin the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt index aec7afd9278..a75d9030c4d 100644 --- a/crates/syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt +++ b/crates/syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt @@ -1,2 +1,2 @@ -RAW_BYTE_STRING 4 "br##" +BYTE_STRING 4 "br##" > error0..4 token("br##") msg(Missing `"` symbol after `#` symbols to begin the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt b/crates/syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt index e22fe5374f0..516e0b78ed0 100644 --- a/crates/syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt +++ b/crates/syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt @@ -1,4 +1,4 @@ -RAW_STRING 4 "r## " +STRING 4 "r## " IDENT 1 "I" WHITESPACE 1 " " IDENT 4 "lack" diff --git a/crates/syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt b/crates/syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt index d74ea4c2796..2f8a6f5f29f 100644 --- a/crates/syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt +++ b/crates/syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt @@ -1,4 +1,4 @@ -RAW_BYTE_STRING 5 "br## " +BYTE_STRING 5 "br## " IDENT 1 "I" WHITESPACE 1 " " IDENT 4 "lack" diff --git a/crates/syntax/test_data/lexer/ok/0008_byte_strings.txt b/crates/syntax/test_data/lexer/ok/0008_byte_strings.txt index bc03b51a8ab..e61ad99bef5 100644 --- a/crates/syntax/test_data/lexer/ok/0008_byte_strings.txt +++ b/crates/syntax/test_data/lexer/ok/0008_byte_strings.txt @@ -4,13 +4,13 @@ BYTE 4 "b\'x\'" WHITESPACE 1 " " BYTE_STRING 6 "b\"foo\"" WHITESPACE 1 " " -RAW_BYTE_STRING 4 "br\"\"" +BYTE_STRING 4 "br\"\"" WHITESPACE 1 "\n" BYTE 6 "b\'\'suf" WHITESPACE 1 " " BYTE_STRING 5 "b\"\"ix" WHITESPACE 1 " " -RAW_BYTE_STRING 6 "br\"\"br" +BYTE_STRING 6 "br\"\"br" WHITESPACE 1 "\n" BYTE 5 "b\'\\n\'" WHITESPACE 1 " " diff --git a/crates/syntax/test_data/lexer/ok/0009_strings.txt b/crates/syntax/test_data/lexer/ok/0009_strings.txt index 4cb4d711d67..988a8877bd7 100644 --- a/crates/syntax/test_data/lexer/ok/0009_strings.txt +++ b/crates/syntax/test_data/lexer/ok/0009_strings.txt @@ -1,6 +1,6 @@ STRING 7 "\"hello\"" WHITESPACE 1 " " -RAW_STRING 8 "r\"world\"" +STRING 8 "r\"world\"" WHITESPACE 1 " " STRING 17 "\"\\n\\\"\\\\no escape\"" WHITESPACE 1 " " diff --git a/crates/syntax/test_data/lexer/ok/0013_raw_strings.txt b/crates/syntax/test_data/lexer/ok/0013_raw_strings.txt index 9cf0957d1e6..db0d5ffd143 100644 --- a/crates/syntax/test_data/lexer/ok/0013_raw_strings.txt +++ b/crates/syntax/test_data/lexer/ok/0013_raw_strings.txt @@ -1,2 +1,2 @@ -RAW_STRING 36 "r###\"this is a r##\"raw\"## string\"###" +STRING 36 "r###\"this is a r##\"raw\"## string\"###" WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/parser/inline/ok/0085_expr_literals.rast b/crates/syntax/test_data/parser/inline/ok/0085_expr_literals.rast index 9a87b5b936d..ae838105d05 100644 --- a/crates/syntax/test_data/parser/inline/ok/0085_expr_literals.rast +++ b/crates/syntax/test_data/parser/inline/ok/0085_expr_literals.rast @@ -104,7 +104,7 @@ SOURCE_FILE@0..189 EQ@142..143 "=" WHITESPACE@143..144 " " LITERAL@144..148 - RAW_STRING@144..148 "r\"d\"" + STRING@144..148 "r\"d\"" SEMICOLON@148..149 ";" WHITESPACE@149..154 "\n " LET_STMT@154..167 @@ -128,7 +128,7 @@ SOURCE_FILE@0..189 EQ@178..179 "=" WHITESPACE@179..180 " " LITERAL@180..185 - RAW_BYTE_STRING@180..185 "br\"f\"" + BYTE_STRING@180..185 "br\"f\"" SEMICOLON@185..186 ";" WHITESPACE@186..187 "\n" R_CURLY@187..188 "}" diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index adc191254d8..8ceaaf60eb4 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -71,16 +71,7 @@ pub(crate) struct KindsSrc<'a> { "trait", "true", "try", "type", "unsafe", "use", "where", "while", ], contextual_keywords: &["auto", "default", "existential", "union", "raw"], - literals: &[ - "INT_NUMBER", - "FLOAT_NUMBER", - "CHAR", - "BYTE", - "STRING", - "RAW_STRING", - "BYTE_STRING", - "RAW_BYTE_STRING", - ], + literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"], tokens: &[ "ERROR", "IDENT", diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 87c934e6676..44460effa83 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -505,7 +505,7 @@ fn ty(&self) -> proc_macro2::Ident { fn lower(grammar: &Grammar) -> AstSrc { let mut res = AstSrc::default(); - res.tokens = "Whitespace Comment String RawString IntNumber FloatNumber" + res.tokens = "Whitespace Comment String ByteString IntNumber FloatNumber" .split_ascii_whitespace() .map(|it| it.to_string()) .collect::>(); -- 2.44.0