fix(prisma-fmt): use UTF-16 offset in the response for the schema that contains multi-byte characters (#4815)

Updated offsets to be UTF-16. This means that our spans will no-longer de-sync with the schema when we run into schemas containing multibyte characters

---------

Co-authored-by: Alexey Orlenko <[email protected]>

Author: key-moon

prisma-fmt/src/code_actions.rs

@@ -4,6 +4,7 @@ mod multi_schema;
 mod relation_mode;
 mod relations;
 
+use crate::offsets::{position_after_span, range_to_span, span_to_range};
 use log::warn;
 use lsp_types::{CodeActionOrCommand, CodeActionParams, Diagnostic, Range, TextEdit, Url, WorkspaceEdit};
 use psl::{
@@ -47,7 +48,7 @@ impl<'a> CodeActionsContext<'a> {
     #[track_caller]
     pub(super) fn diagnostics_for_span(&self, span: ast::Span) -> impl Iterator<Item = &Diagnostic> {
         self.diagnostics().iter().filter(move |diag| {
-            span.overlaps(crate::range_to_span(
+            span.overlaps(range_to_span(
                 diag.range,
                 self.initiating_file_source(),
                 self.initiating_file_id,
@@ -167,7 +168,7 @@ fn create_missing_attribute<'a>(
         let new_text = format!(" @{attribute_name}");
 
         let field = fields.next().unwrap();
-        let position = crate::position_after_span(field.ast_field().span(), schema);
+        let position = position_after_span(field.ast_field().span(), schema);
 
         let range = Range {
             start: position,
@@ -187,25 +188,15 @@ fn create_missing_attribute<'a>(
             &model.ast_model().attributes,
         );
 
-        let range = range_after_span(schema, model.ast_model().span());
+        let range = range_after_span(model.ast_model().span(), schema);
         (formatted_attribute, range)
     };
 
     TextEdit { range, new_text }
 }
 
-fn range_after_span(schema: &str, span: Span) -> Range {
-    let start = crate::offset_to_position(span.end - 1, schema);
-    let end = crate::offset_to_position(span.end, schema);
-
-    Range { start, end }
-}
-
-fn span_to_range(schema: &str, span: Span) -> Range {
-    let start = crate::offset_to_position(span.start, schema);
-    let end = crate::offset_to_position(span.end, schema);
-
-    Range { start, end }
+fn range_after_span(span: Span, schema: &str) -> Range {
+    span_to_range(Span::new(span.end - 1, span.end, span.file_id), schema)
 }
 
 fn format_field_attribute(attribute: &str) -> String {
@@ -253,8 +244,8 @@ fn create_text_edit(
     span: Span,
 ) -> Result<WorkspaceEdit, Box<dyn std::error::Error>> {
     let range = match append {
-        true => range_after_span(target_file_content, span),
-        false => span_to_range(target_file_content, span),
+        true => range_after_span(span, target_file_content),
+        false => span_to_range(span, target_file_content),
     };
 
     let text = TextEdit {

prisma-fmt/src/code_actions/block.rs

@@ -28,7 +28,7 @@ pub(super) fn create_missing_block_for_model(
         file_id: span_model.file_id,
     };
 
-    let range = super::range_after_span(context.initiating_file_source(), span);
+    let range = super::range_after_span(span, context.initiating_file_source());
 
     diagnostics.iter().for_each(|diag| {
         push_missing_block(
@@ -83,7 +83,7 @@ pub(super) fn create_missing_block_for_type(
         file_id: span_type.file_id,
     };
 
-    let range = super::range_after_span(context.initiating_file_source(), span);
+    let range = super::range_after_span(span, context.initiating_file_source());
     diagnostics.iter().for_each(|diag| {
         push_missing_block(
             diag,

prisma-fmt/src/code_actions/relations.rs

@@ -271,7 +271,7 @@ pub(super) fn add_index_for_relation_fields(
         &relation.model().ast_model().attributes,
     );
 
-    let range = super::range_after_span(context.initiating_file_source(), relation.model().ast_model().span());
+    let range = super::range_after_span(relation.model().ast_model().span(), context.initiating_file_source());
     let text = TextEdit {
         range,
         new_text: formatted_attribute,

prisma-fmt/src/lib.rs

@@ -5,14 +5,14 @@ mod get_dmmf;
 mod lint;
 mod merge_schemas;
 mod native;
+mod offsets;
 mod preview;
 mod schema_file_input;
 mod text_document_completion;
 mod validate;
 
 use log::*;
-use lsp_types::{Position, Range};
-use psl::{diagnostics::FileId, parser_database::ast};
+pub use offsets::span_to_range;
 use schema_file_input::SchemaFileInput;
 
 /// The API is modelled on an LSP [completion
@@ -221,132 +221,3 @@ pub fn get_config(get_config_params: String) -> String {
 pub fn get_dmmf(get_dmmf_params: String) -> Result<String, String> {
     get_dmmf::get_dmmf(&get_dmmf_params)
 }
-
-/// The LSP position is expressed as a (line, col) tuple, but our pest-based parser works with byte
-/// offsets. This function converts from an LSP position to a pest byte offset. Returns `None` if
-/// the position has a line past the end of the document, or a character position past the end of
-/// the line.
-pub(crate) fn position_to_offset(position: &Position, document: &str) -> Option<usize> {
-    let mut offset = 0;
-    let mut line_offset = position.line;
-    let mut character_offset = position.character;
-    let mut chars = document.chars();
-
-    while line_offset > 0 {
-        loop {
-            match chars.next() {
-                Some('\n') => {
-                    offset += 1;
-                    break;
-                }
-                Some(_) => {
-                    offset += 1;
-                }
-                None => return Some(offset),
-            }
-        }
-
-        line_offset -= 1;
-    }
-
-    while character_offset > 0 {
-        match chars.next() {
-            Some('\n') | None => return Some(offset),
-            Some(_) => {
-                offset += 1;
-                character_offset -= 1;
-            }
-        }
-    }
-
-    Some(offset)
-}
-
-#[track_caller]
-/// Converts an LSP range to a span.
-pub(crate) fn range_to_span(range: Range, document: &str, file_id: FileId) -> ast::Span {
-    let start = position_to_offset(&range.start, document).unwrap();
-    let end = position_to_offset(&range.end, document).unwrap();
-
-    ast::Span::new(start, end, file_id)
-}
-
-/// Gives the LSP position right after the given span, skipping any trailing newlines
-pub(crate) fn position_after_span(span: ast::Span, document: &str) -> Position {
-    let end = match (document.chars().nth(span.end - 2), document.chars().nth(span.end - 1)) {
-        (Some('\r'), Some('\n')) => span.end - 2,
-        (_, Some('\n')) => span.end - 1,
-        _ => span.end,
-    };
-
-    offset_to_position(end, document)
-}
-
-/// Converts a byte offset to an LSP position, if the given offset
-/// does not overflow the document.
-pub fn offset_to_position(offset: usize, document: &str) -> Position {
-    let mut position = Position::default();
-
-    for (i, chr) in document.chars().enumerate() {
-        match chr {
-            _ if i == offset => {
-                return position;
-            }
-            '\n' => {
-                position.character = 0;
-                position.line += 1;
-            }
-            _ => {
-                position.character += 1;
-            }
-        }
-    }
-
-    position
-}
-
-#[cfg(test)]
-mod tests {
-    use lsp_types::Position;
-    use psl::diagnostics::{FileId, Span};
-
-    use crate::position_after_span;
-
-    // On Windows, a newline is actually two characters.
-    #[test]
-    fn position_to_offset_with_crlf() {
-        let schema = "\r\nmodel Test {\r\n    id Int @id\r\n}";
-        // Let's put the cursor on the "i" in "id Int".
-        let expected_offset = schema.chars().position(|c| c == 'i').unwrap();
-        let found_offset = super::position_to_offset(&Position { line: 2, character: 4 }, schema).unwrap();
-
-        assert_eq!(found_offset, expected_offset);
-    }
-
-    #[test]
-    fn position_after_span_no_newline() {
-        let str = "some string";
-        let span = Span::new(0, str.len(), FileId::ZERO);
-        let pos = position_after_span(span, str);
-        assert_eq!(pos.line, 0);
-        assert_eq!(pos.character, 11);
-    }
-
-    #[test]
-    fn position_after_span_lf() {
-        let str = "some string\n";
-        let span = Span::new(0, str.len(), FileId::ZERO);
-        let pos = position_after_span(span, str);
-        assert_eq!(pos.line, 0);
-        assert_eq!(pos.character, 11);
-    }
-
-    #[test]
-    fn position_after_span_crlf() {
-        let str = "some string\r\n";
-        let span = Span::new(0, str.len(), FileId::ZERO);
-        let pos = position_after_span(span, str);
-        assert_eq!(pos.line, 0);
-        assert_eq!(pos.character, 11);
-    }
-}

prisma-fmt/src/lint.rs

@@ -1,4 +1,8 @@
-use psl::diagnostics::{DatamodelError, DatamodelWarning};
+use crate::offsets::span_to_lsp_offsets;
+use psl::{
+    diagnostics::{DatamodelError, DatamodelWarning},
+    ValidatedSchema,
+};
 
 use crate::schema_file_input::SchemaFileInput;
 
@@ -12,33 +16,39 @@ pub struct MiniError {
 }
 
 pub(crate) fn run(schema: SchemaFileInput) -> String {
-    let schema = match schema {
+    let validated_schema = match schema {
         SchemaFileInput::Single(file) => psl::validate(file.into()),
         SchemaFileInput::Multiple(files) => psl::validate_multi_file(&files),
     };
-    let diagnostics = &schema.diagnostics;
+    let ValidatedSchema { diagnostics, db, .. } = &validated_schema;
 
     let mut mini_errors: Vec<MiniError> = diagnostics
         .errors()
         .iter()
-        .map(|err: &DatamodelError| MiniError {
-            file_name: schema.db.file_name(err.span().file_id).to_owned(),
-            start: err.span().start,
-            end: err.span().end,
-            text: err.message().to_string(),
-            is_warning: false,
+        .map(|err: &DatamodelError| {
+            let (start, end) = span_to_lsp_offsets(err.span(), db.source(err.span().file_id));
+            MiniError {
+                file_name: db.file_name(err.span().file_id).to_owned(),
+                start,
+                end,
+                text: err.message().to_string(),
+                is_warning: false,
+            }
         })
         .collect();
 
     let mut mini_warnings: Vec<MiniError> = diagnostics
         .warnings()
         .iter()
-        .map(|warn: &DatamodelWarning| MiniError {
-            file_name: schema.db.file_name(warn.span().file_id).to_owned(),
-            start: warn.span().start,
-            end: warn.span().end,
-            text: warn.message().to_owned(),
-            is_warning: true,
+        .map(|warn: &DatamodelWarning| {
+            let (start, end) = span_to_lsp_offsets(warn.span(), db.source(warn.span().file_id));
+            MiniError {
+                file_name: db.file_name(warn.span().file_id).to_owned(),
+                start,
+                end,
+                text: warn.message().to_owned(),
+                is_warning: true,
+            }
         })
         .collect();
 
@@ -64,6 +74,28 @@ mod tests {
         serde_json::to_string_pretty(&value).unwrap()
     }
 
+    #[test]
+    fn should_return_utf16_offset() {
+        let schema = indoc! {r#"
+            // 🌐 ｍｕｌｔｉｂｙｔｅ
+            😀
+        "#};
+        let datamodel = SchemaFileInput::Single(schema.to_string());
+
+        let expected = expect![[r#"
+            [
+              {
+                "file_name": "schema.prisma",
+                "start": 16,
+                "end": 19,
+                "text": "Error validating: This line is invalid. It does not start with any known Prisma schema keyword.",
+                "is_warning": false
+              }
+            ]"#]];
+
+        expected.assert_eq(&lint(datamodel));
+    }
+
     #[test]
     fn single_deprecated_preview_features_should_give_a_warning() {
         let schema = indoc! {r#"

prisma-fmt/src/main.rs

@@ -1,6 +1,5 @@
 mod actions;
 mod format;
-// mod lint;
 mod native;
 mod preview;
 mod schema_file_input;