Merge #7657

7657: utf8 r=matklad a=matklad

- Prepare for utf-8 offsets
- reduce code duplication in tests
- Make utf8 default, implement utf16 in terms of it
- Make it easy to add additional context for offset conversion
- Implement utf8 offsets

closes #7453

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

diff --combined crates/rust-analyzer/src/config.rs
index 04a77d677b671c789b1bce80ef06f76c600e5743,9c873c097b1fae0ebf64eaef97bdbdd85a1bff58..556fc2eeb6e235c38e806d54969a8de6909bbcc3
--- 1/crates/rust-analyzer/src/config.rs
--- 2/crates/rust-analyzer/src/config.rs
+++ b/crates/rust-analyzer/src/config.rs
@@@ -23,7 -23,10 +23,10 @@@ use rustc_hash::FxHashSet
  use serde::{de::DeserializeOwned, Deserialize};
  use vfs::AbsPathBuf;
  
- use crate::{caps::completion_item_edit_resolve, diagnostics::DiagnosticsMapConfig};
+ use crate::{
+     caps::completion_item_edit_resolve, diagnostics::DiagnosticsMapConfig,
+     line_index::OffsetEncoding, lsp_ext::supports_utf8,
+ };
  
  config_data! {
      struct ConfigData {
@@@ -415,6 -418,13 +418,13 @@@ impl Config 
              false
          )
      }
+     pub fn offset_encoding(&self) -> OffsetEncoding {
+         if supports_utf8(&self.caps) {
+             OffsetEncoding::Utf8
+         } else {
+             OffsetEncoding::Utf16
+         }
+     }
  
      fn experimental(&self, index: &'static str) -> bool {
          try_or!(self.caps.experimental.as_ref()?.get(index)?.as_bool()?, false)
@@@ -519,7 -529,7 +529,7 @@@
                      .data
                      .checkOnSave_target
                      .clone()
 -                    .or(self.data.cargo_target.clone()),
 +                    .or_else(|| self.data.cargo_target.clone()),
                  all_targets: self.data.checkOnSave_allTargets,
                  no_default_features: self
                      .data
@@@ -533,7 -543,7 +543,7 @@@
                      .data
                      .checkOnSave_features
                      .clone()
 -                    .unwrap_or(self.data.cargo_features.clone()),
 +                    .unwrap_or_else(|| self.data.cargo_features.clone()),
                  extra_args: self.data.checkOnSave_extraArgs.clone(),
              },
          };
@@@ -731,7 -741,7 +741,7 @@@ fn get_field<T: DeserializeOwned>
  fn schema(fields: &[(&'static str, &'static str, &[&str], &str)]) -> serde_json::Value {
      for ((f1, ..), (f2, ..)) in fields.iter().zip(&fields[1..]) {
          fn key(f: &str) -> &str {
 -            f.splitn(2, "_").next().unwrap()
 +            f.splitn(2, '_').next().unwrap()
          }
          assert!(key(f1) <= key(f2), "wrong field order: {:?} {:?}", f1, f2);
      }

diff --combined crates/rust-analyzer/src/to_proto.rs
index b0ddb603dbe668e9e12505b638a86b17ee70ce03,6aff65575dcdeb038e852adfe9d5589b4505fc6c..70cb7fbab5c6820fdb27947e689fc79b7d680975
--- 1/crates/rust-analyzer/src/to_proto.rs
--- 2/crates/rust-analyzer/src/to_proto.rs
+++ b/crates/rust-analyzer/src/to_proto.rs
@@@ -7,22 -7,29 +7,29 @@@ use std::
  use ide::{
      Annotation, AnnotationKind, Assist, AssistKind, CallInfo, CompletionItem, CompletionItemKind,
      Documentation, FileId, FileRange, FileSystemEdit, Fold, FoldKind, Highlight, HlMod, HlPunct,
-     HlRange, HlTag, Indel, InlayHint, InlayKind, InsertTextFormat, LineIndex, Markup,
-     NavigationTarget, ReferenceAccess, RenameError, Runnable, Severity, SourceChange, TextEdit,
-     TextRange, TextSize,
+     HlRange, HlTag, Indel, InlayHint, InlayKind, InsertTextFormat, Markup, NavigationTarget,
+     ReferenceAccess, RenameError, Runnable, Severity, SourceChange, TextEdit, TextRange, TextSize,
  };
  use ide_db::SymbolKind;
  use itertools::Itertools;
  use serde_json::to_value;
  
  use crate::{
-     cargo_target_spec::CargoTargetSpec, global_state::GlobalStateSnapshot,
-     line_endings::LineEndings, lsp_ext, semantic_tokens, Result,
+     cargo_target_spec::CargoTargetSpec,
+     global_state::GlobalStateSnapshot,
+     line_index::{LineEndings, LineIndex, OffsetEncoding},
+     lsp_ext, semantic_tokens, Result,
  };
  
  pub(crate) fn position(line_index: &LineIndex, offset: TextSize) -> lsp_types::Position {
-     let line_col = line_index.line_col(offset);
-     lsp_types::Position::new(line_col.line, line_col.col_utf16)
+     let line_col = line_index.index.line_col(offset);
+     match line_index.encoding {
+         OffsetEncoding::Utf8 => lsp_types::Position::new(line_col.line, line_col.col),
+         OffsetEncoding::Utf16 => {
+             let line_col = line_index.index.to_utf16(line_col);
+             lsp_types::Position::new(line_col.line, line_col.col)
+         }
+     }
  }
  
  pub(crate) fn range(line_index: &LineIndex, range: TextRange) -> lsp_types::Range {
@@@ -122,13 -129,9 +129,9 @@@ pub(crate) fn completion_item_kind
      }
  }
  
- pub(crate) fn text_edit(
-     line_index: &LineIndex,
-     line_endings: LineEndings,
-     indel: Indel,
- ) -> lsp_types::TextEdit {
+ pub(crate) fn text_edit(line_index: &LineIndex, indel: Indel) -> lsp_types::TextEdit {
      let range = range(line_index, indel.delete);
-     let new_text = match line_endings {
+     let new_text = match line_index.endings {
          LineEndings::Unix => indel.insert,
          LineEndings::Dos => indel.insert.replace('\n', "\r\n"),
      };
@@@ -137,11 -140,10 +140,10 @@@
  
  pub(crate) fn snippet_text_edit(
      line_index: &LineIndex,
-     line_endings: LineEndings,
      is_snippet: bool,
      indel: Indel,
  ) -> lsp_ext::SnippetTextEdit {
-     let text_edit = text_edit(line_index, line_endings, indel);
+     let text_edit = text_edit(line_index, indel);
      let insert_text_format =
          if is_snippet { Some(lsp_types::InsertTextFormat::Snippet) } else { None };
      lsp_ext::SnippetTextEdit {
@@@ -153,27 -155,24 +155,24 @@@
  
  pub(crate) fn text_edit_vec(
      line_index: &LineIndex,
-     line_endings: LineEndings,
      text_edit: TextEdit,
  ) -> Vec<lsp_types::TextEdit> {
-     text_edit.into_iter().map(|indel| self::text_edit(line_index, line_endings, indel)).collect()
+     text_edit.into_iter().map(|indel| self::text_edit(line_index, indel)).collect()
  }
  
  pub(crate) fn snippet_text_edit_vec(
      line_index: &LineIndex,
-     line_endings: LineEndings,
      is_snippet: bool,
      text_edit: TextEdit,
  ) -> Vec<lsp_ext::SnippetTextEdit> {
      text_edit
          .into_iter()
-         .map(|indel| self::snippet_text_edit(line_index, line_endings, is_snippet, indel))
+         .map(|indel| self::snippet_text_edit(line_index, is_snippet, indel))
          .collect()
  }
  
  pub(crate) fn completion_item(
      line_index: &LineIndex,
-     line_endings: LineEndings,
      completion_item: CompletionItem,
  ) -> Vec<lsp_types::CompletionItem> {
      fn set_score(res: &mut lsp_types::CompletionItem, label: &str) {
@@@ -190,19 -189,19 +189,19 @@@
      for indel in completion_item.text_edit().iter() {
          if indel.delete.contains_range(source_range) {
              text_edit = Some(if indel.delete == source_range {
-                 self::text_edit(line_index, line_endings, indel.clone())
+                 self::text_edit(line_index, indel.clone())
              } else {
                  assert!(source_range.end() == indel.delete.end());
                  let range1 = TextRange::new(indel.delete.start(), source_range.start());
                  let range2 = source_range;
                  let indel1 = Indel::replace(range1, String::new());
                  let indel2 = Indel::replace(range2, indel.insert.clone());
-                 additional_text_edits.push(self::text_edit(line_index, line_endings, indel1));
-                 self::text_edit(line_index, line_endings, indel2)
+                 additional_text_edits.push(self::text_edit(line_index, indel1));
+                 self::text_edit(line_index, indel2)
              })
          } else {
              assert!(source_range.intersect(indel.delete).is_none());
-             let text_edit = self::text_edit(line_index, line_endings, indel.clone());
+             let text_edit = self::text_edit(line_index, indel.clone());
              additional_text_edits.push(text_edit);
          }
      }
@@@ -358,7 -357,7 +357,7 @@@ pub(crate) fn semantic_tokens
          let token_index = semantic_tokens::type_index(type_);
          let modifier_bitset = mods.0;
  
-         for mut text_range in line_index.lines(highlight_range.range) {
+         for mut text_range in line_index.index.lines(highlight_range.range) {
              if text[text_range].ends_with('\n') {
                  text_range =
                      TextRange::new(text_range.start(), text_range.end() - TextSize::of('\n'));
@@@ -565,7 -564,7 +564,7 @@@ pub(crate) fn location
      frange: FileRange,
  ) -> Result<lsp_types::Location> {
      let url = url(snap, frange.file_id);
-     let line_index = snap.analysis.file_line_index(frange.file_id)?;
+     let line_index = snap.file_line_index(frange.file_id)?;
      let range = range(&line_index, frange.range);
      let loc = lsp_types::Location::new(url, range);
      Ok(loc)
@@@ -577,7 -576,7 +576,7 @@@ pub(crate) fn location_from_nav
      nav: NavigationTarget,
  ) -> Result<lsp_types::Location> {
      let url = url(snap, nav.file_id);
-     let line_index = snap.analysis.file_line_index(nav.file_id)?;
+     let line_index = snap.file_line_index(nav.file_id)?;
      let range = range(&line_index, nav.full_range);
      let loc = lsp_types::Location::new(url, range);
      Ok(loc)
@@@ -590,7 -589,7 +589,7 @@@ pub(crate) fn location_link
  ) -> Result<lsp_types::LocationLink> {
      let origin_selection_range = match src {
          Some(src) => {
-             let line_index = snap.analysis.file_line_index(src.file_id)?;
+             let line_index = snap.file_line_index(src.file_id)?;
              let range = range(&line_index, src.range);
              Some(range)
          }
@@@ -610,7 -609,7 +609,7 @@@ fn location_info
      snap: &GlobalStateSnapshot,
      target: NavigationTarget,
  ) -> Result<(lsp_types::Url, lsp_types::Range, lsp_types::Range)> {
-     let line_index = snap.analysis.file_line_index(target.file_id)?;
+     let line_index = snap.file_line_index(target.file_id)?;
  
      let target_uri = url(snap, target.file_id);
      let target_range = range(&line_index, target.full_range);
@@@ -648,12 -647,8 +647,8 @@@ pub(crate) fn snippet_text_document_edi
      edit: TextEdit,
  ) -> Result<lsp_ext::SnippetTextDocumentEdit> {
      let text_document = optional_versioned_text_document_identifier(snap, file_id);
-     let line_index = snap.analysis.file_line_index(file_id)?;
-     let line_endings = snap.file_line_endings(file_id);
-     let edits = edit
-         .into_iter()
-         .map(|it| snippet_text_edit(&line_index, line_endings, is_snippet, it))
-         .collect();
+     let line_index = snap.file_line_index(file_id)?;
+     let edits = edit.into_iter().map(|it| snippet_text_edit(&line_index, is_snippet, it)).collect();
      Ok(lsp_ext::SnippetTextDocumentEdit { text_document, edits })
  }
  
@@@ -674,9 -669,8 +669,8 @@@ pub(crate) fn snippet_text_document_ops
              if !initial_contents.is_empty() {
                  let text_document =
                      lsp_types::OptionalVersionedTextDocumentIdentifier { uri, version: None };
-                 let range = range(&LineIndex::new(""), TextRange::empty(TextSize::from(0)));
                  let text_edit = lsp_ext::SnippetTextEdit {
-                     range,
+                     range: lsp_types::Range::default(),
                      new_text: initial_contents,
                      insert_text_format: Some(lsp_types::InsertTextFormat::PlainText),
                  };
@@@ -867,7 -861,7 +861,7 @@@ pub(crate) fn code_lens
  ) -> Result<lsp_types::CodeLens> {
      match annotation.kind {
          AnnotationKind::Runnable { debug, runnable: run } => {
-             let line_index = snap.analysis.file_line_index(run.nav.file_id)?;
+             let line_index = snap.file_line_index(run.nav.file_id)?;
              let annotation_range = range(&line_index, annotation.range);
  
              let action = run.action();
@@@ -883,7 -877,7 +877,7 @@@
              Ok(lsp_types::CodeLens { range: annotation_range, command: Some(command), data: None })
          }
          AnnotationKind::HasImpls { position: file_position, data } => {
-             let line_index = snap.analysis.file_line_index(file_position.file_id)?;
+             let line_index = snap.file_line_index(file_position.file_id)?;
              let annotation_range = range(&line_index, annotation.range);
              let url = url(snap, file_position.file_id);
  
@@@ -891,10 -885,10 +885,10 @@@
  
              let id = lsp_types::TextDocumentIdentifier { uri: url.clone() };
  
 -            let doc_pos = lsp_types::TextDocumentPositionParams::new(id.clone(), position);
 +            let doc_pos = lsp_types::TextDocumentPositionParams::new(id, position);
  
              let goto_params = lsp_types::request::GotoImplementationParams {
 -                text_document_position_params: doc_pos.clone(),
 +                text_document_position_params: doc_pos,
                  work_done_progress_params: Default::default(),
                  partial_result_params: Default::default(),
              };
@@@ -926,7 -920,7 +920,7 @@@
              })
          }
          AnnotationKind::HasReferences { position: file_position, data } => {
-             let line_index = snap.analysis.file_line_index(file_position.file_id)?;
+             let line_index = snap.file_line_index(file_position.file_id)?;
              let annotation_range = range(&line_index, annotation.range);
              let url = url(snap, file_position.file_id);
  
@@@ -1060,6 -1054,8 +1054,8 @@@ pub(crate) fn rename_error(err: RenameE
  
  #[cfg(test)]
  mod tests {
+     use std::sync::Arc;
+ 
      use hir::PrefixKind;
      use ide::Analysis;
      use ide_db::helpers::{insert_use::InsertUseConfig, SnippetCap};
@@@ -1077,7 -1073,11 +1073,11 @@@
          }"#;
  
          let (offset, text) = test_utils::extract_offset(fixture);
-         let line_index = LineIndex::new(&text);
+         let line_index = LineIndex {
+             index: Arc::new(ide::LineIndex::new(&text)),
+             endings: LineEndings::Unix,
+             encoding: OffsetEncoding::Utf16,
+         };
          let (analysis, file_id) = Analysis::from_single_file(text);
          let completions: Vec<(String, Option<String>)> = analysis
              .completions(
@@@ -1095,7 -1095,7 +1095,7 @@@
              .unwrap()
              .into_iter()
              .filter(|c| c.label().ends_with("arg"))
-             .map(|c| completion_item(&line_index, LineEndings::Unix, c))
+             .map(|c| completion_item(&line_index, c))
              .flat_map(|comps| comps.into_iter().map(|c| (c.label, c.sort_text)))
              .collect();
          expect_test::expect![[r#"
@@@ -1133,7 -1133,11 +1133,11 @@@ fn main() 
          let folds = analysis.folding_ranges(file_id).unwrap();
          assert_eq!(folds.len(), 4);
  
-         let line_index = LineIndex::new(&text);
+         let line_index = LineIndex {
+             index: Arc::new(ide::LineIndex::new(&text)),
+             endings: LineEndings::Unix,
+             encoding: OffsetEncoding::Utf16,
+         };
          let converted: Vec<lsp_types::FoldingRange> =
              folds.into_iter().map(|it| folding_range(&text, &line_index, true, it)).collect();