package rustlang import ( "strings" "testing" "rickub.com/turbo-editors/turbo-core/syntax" ) // classAt returns the class covering a rune column on a line, and whether any // span covers it at all. It is how nearly every test below asks its question. func classAt(spans [][]syntax.Span, line, col int) (syntax.Class, bool) { if line < 0 || line >= len(spans) { return 0, false } for _, s := range spans[line] { if col >= s.Start && col < s.End { return s.Class, true } } return 0, false } // classOfFirst returns the class of the first occurrence of word in src. func classOfFirst(t *testing.T, src, word string) syntax.Class { t.Helper() index := strings.Index(src, word) if index < 0 { t.Fatalf("%q does not appear in the source", word) } line := strings.Count(src[:index], "\n") col := index - (strings.LastIndex(src[:index], "\n") + 1) class, ok := classAt(Highlight(src), line, col) if !ok { t.Fatalf("no span covers %q at line %d column %d", word, line, col) } return class } func TestHighlightReturnsOneEntryPerLine(t *testing.T) { // The editor indexes the result by line number without checking, so a short // result is an index out of range in the middle of a redraw. tests := []struct { name string src string want int }{ {"empty", "", 1}, {"one line without a terminator", "fn main() {}", 1}, {"one line with a terminator", "fn main() {}\n", 2}, {"three lines", "a\nb\nc", 3}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { if got := len(Highlight(tc.src)); got != tc.want { t.Errorf("Highlight(%q) returned %d lines, want %d", tc.src, got, tc.want) } }) } } func TestEachTokenClass(t *testing.T) { const src = `use std::fmt; // a comment /// a doc comment struct Point { x: i32, } fn main() { let name = "world"; let initial = 'w'; let count = 42; let ratio = 1.5; println!("hello {name}"); } ` tests := []struct { word string want syntax.Class }{ {"use", syntax.ClassKeyword}, {"struct", syntax.ClassKeyword}, {"fn", syntax.ClassKeyword}, {"let", syntax.ClassKeyword}, {"// a comment", syntax.ClassComment}, {"/// a doc comment", syntax.ClassComment}, {"Point", syntax.ClassType}, {"i32", syntax.ClassType}, {`"world"`, syntax.ClassString}, {"'w'", syntax.ClassChar}, {"42", syntax.ClassNumber}, {"1.5", syntax.ClassNumber}, {"println!", syntax.ClassBuiltin}, {"::", syntax.ClassPunctuation}, } for _, test := range tests { t.Run(test.word, func(t *testing.T) { if got := classOfFirst(t, src, test.word); got != test.want { t.Errorf("%q is %v, want %v", test.word, got, test.want) } }) } } func TestADeclaredFunctionNameIsAFunction(t *testing.T) { if got := classOfFirst(t, "fn parse(input: &str) {}", "parse"); got != syntax.ClassFunction { t.Errorf("parse is %v, want function", got) } } func TestACallIsAFunction(t *testing.T) { if got := classOfFirst(t, "let n = compute(3);", "compute"); got != syntax.ClassFunction { t.Errorf("compute is %v, want function", got) } } func TestAnUpperCaseNameIsAType(t *testing.T) { // Rust's naming convention is strong enough to lean on: a type, a trait and // an enum variant are all UpperCamelCase, and nothing else is. for _, word := range []string{"HashMap", "Display", "MyError"} { src := "let x: " + word + " = todo();" if got := classOfFirst(t, src, word); got != syntax.ClassType { t.Errorf("%s is %v, want type", word, got) } } } func TestALowerCaseNameIsAPlainIdentifier(t *testing.T) { if got := classOfFirst(t, "let total = subtotal + tax;", "subtotal"); got != syntax.ClassIdentifier { t.Errorf("subtotal is %v, want identifier", got) } } func TestAPrimitiveTypeOutranksTheCallHeuristic(t *testing.T) { // u8::from_str_radix has a "(" after it eventually, but u8 is a type. if got := classOfFirst(t, "let n = u8::MAX;", "u8"); got != syntax.ClassType { t.Errorf("u8 is %v, want type", got) } } func TestSelfAndCapitalSelfAreTypes(t *testing.T) { const src = "impl Point {\n fn x(&self) -> Self { *self }\n}" if got := classOfFirst(t, src, "self"); got != syntax.ClassType { t.Errorf("self is %v, want type", got) } if got := classOfFirst(t, src, "Self"); got != syntax.ClassType { t.Errorf("Self is %v, want type", got) } } func TestTheLiteralsAreConstants(t *testing.T) { for _, word := range []string{"true", "false", "None", "Some", "Ok", "Err"} { src := "let v = " + word + ";" if got := classOfFirst(t, src, word); got != syntax.ClassConstant { t.Errorf("%s is %v, want constant", word, got) } } } func TestAMacroTakesItsExclamationMarkWithIt(t *testing.T) { // println! is one name; colouring the ! separately would read as a negation. spans := Highlight(`println!("hi");`) class, ok := classAt(spans, 0, 7) // the "!" if !ok || class != syntax.ClassBuiltin { t.Errorf("the ! of println! is %v (covered: %v), want builtin", class, ok) } } func TestNotEqualsIsNotAMacro(t *testing.T) { // `a != b` has a ! straight after a word, and is not a macro invocation. if got := classOfFirst(t, "if a != b {}", "a"); got == syntax.ClassBuiltin { t.Error("a != b was read as a macro call") } } func TestAnAttributeIsColouredAsOne(t *testing.T) { tests := []string{"#[derive(Debug)]", "#![no_std]", "#[cfg(test)]"} for _, src := range tests { t.Run(src, func(t *testing.T) { spans := Highlight(src + "\nstruct S;") for col := 0; col < len(src); col++ { class, ok := classAt(spans, 0, col) if !ok || class != syntax.ClassAttribute { t.Fatalf("column %d of %q is %v (covered: %v), want attribute", col, src, class, ok) } } }) } } func TestCodeAfterAnAttributeIsStillCode(t *testing.T) { spans := Highlight("#[derive(Debug)] struct S;") class, ok := classAt(spans, 0, strings.Index("#[derive(Debug)] struct S;", "struct")) if !ok || class != syntax.ClassKeyword { t.Errorf("struct after an attribute is %v (covered: %v), want keyword", class, ok) } } func TestBlockCommentsNest(t *testing.T) { // Rust nests them, so a flag instead of a depth would end this comment at // the first */ and colour the rest of the line as code. const src = "/* outer /* inner */ still a comment */ let x = 1;" spans := Highlight(src) commentEnd := strings.Index(src, "*/ let") + 2 for col := 0; col < commentEnd; col++ { class, ok := classAt(spans, 0, col) if !ok || class != syntax.ClassComment { t.Fatalf("column %d is %v (covered: %v), want comment", col, class, ok) } } if got := classOfFirst(t, src, "let"); got != syntax.ClassKeyword { t.Errorf("the code after the comment is %v, want keyword", got) } } func TestABlockCommentCarriesAcrossLines(t *testing.T) { spans := Highlight("/* one\ntwo\n*/ let x = 1;") for line := range 2 { class, ok := classAt(spans, line, 0) if !ok || class != syntax.ClassComment { t.Errorf("line %d is %v (covered: %v), want comment", line, class, ok) } } class, ok := classAt(spans, 2, 3) // the "l" of let if !ok || class != syntax.ClassKeyword { t.Errorf("the code after the comment is %v (covered: %v), want keyword", class, ok) } } func TestANestedCommentCarriesItsDepthAcrossLines(t *testing.T) { spans := Highlight("/* a\n/* b\n*/ still\n*/ let x = 1;") // Line 2 closes only the inner comment, so it is still a comment. class, ok := classAt(spans, 2, 3) if !ok || class != syntax.ClassComment { t.Errorf("line 2 is %v (covered: %v); the outer comment was closed too early", class, ok) } class, ok = classAt(spans, 3, 3) if !ok || class != syntax.ClassKeyword { t.Errorf("line 3 is %v (covered: %v), want the code after the comment", class, ok) } } func TestRawStringsAreColouredToTheirHashes(t *testing.T) { const src = `let re = r#"a "quoted" thing"#;` spans := Highlight(src) // The quote in the middle must not end the string. class, ok := classAt(spans, 0, strings.Index(src, `"quoted"`)) if !ok || class != syntax.ClassString { t.Errorf("the inner quote is %v (covered: %v), want string", class, ok) } if got := classOfFirst(t, src, ";"); got != syntax.ClassPunctuation { t.Errorf("the semicolon after the raw string is %v, want punctuation", got) } } func TestARawStringCarriesAcrossLines(t *testing.T) { spans := Highlight("let q = r#\"select\nfrom t\n\"#;\nlet x = 1;") class, ok := classAt(spans, 1, 0) if !ok || class != syntax.ClassString { t.Errorf("the second line of the raw string is %v (covered: %v), want string", class, ok) } class, ok = classAt(spans, 3, 0) if !ok || class != syntax.ClassKeyword { t.Errorf("the line after the raw string is %v (covered: %v), want code", class, ok) } } func TestARawStringWithNoHashesEndsAtItsQuote(t *testing.T) { const src = `let s = r"plain"; let n = 1;` if got := classOfFirst(t, src, "let n"); got != syntax.ClassKeyword { t.Errorf("the code after r\"plain\" is %v, want keyword", got) } } func TestAnOrdinaryStringMayCrossALine(t *testing.T) { // Rust allows a real newline inside "…", so this is not an error state. spans := Highlight("let s = \"one\ntwo\";\nlet x = 1;") class, ok := classAt(spans, 1, 0) if !ok || class != syntax.ClassString { t.Errorf("the second line of the string is %v (covered: %v), want string", class, ok) } class, ok = classAt(spans, 2, 0) if !ok || class != syntax.ClassKeyword { t.Errorf("the line after the string is %v (covered: %v), want code", class, ok) } } func TestAnEscapedQuoteDoesNotEndAString(t *testing.T) { const src = `let s = "a \" b"; let n = 1;` if got := classOfFirst(t, src, "let n"); got != syntax.ClassKeyword { t.Errorf("the code after an escaped quote is %v, want keyword", got) } } func TestByteStringsAndByteCharacters(t *testing.T) { if got := classOfFirst(t, `let b = b"bytes";`, `b"bytes"`); got != syntax.ClassString { t.Errorf(`b"bytes" is %v, want string`, got) } if got := classOfFirst(t, `let c = b'x';`, `b'x'`); got != syntax.ClassChar { t.Errorf(`b'x' is %v, want char`, got) } } func TestALifetimeIsNotACharacterLiteral(t *testing.T) { // They begin with the same rune, and getting this wrong strings the rest of // the line: 'a is a lifetime, 'a' is a character. const src = "fn longest<'a>(x: &'a str) -> &'a str { x }" class, ok := classAt(Highlight(src), 0, strings.Index(src, "'a>")) if !ok || class == syntax.ClassChar || class == syntax.ClassString { t.Errorf("the lifetime 'a is %v (covered: %v), want it not read as a literal", class, ok) } // The code after it must survive, which is the failure that actually hurts. if got := classOfFirst(t, src, "str"); got != syntax.ClassType { t.Errorf("str after two lifetimes is %v, want type", got) } } func TestStaticIsALifetimeToo(t *testing.T) { const src = "let s: &'static str = \"hi\";" if got := classOfFirst(t, src, `"hi"`); got != syntax.ClassString { t.Errorf("the string after 'static is %v, want string; the lifetime swallowed it", got) } } func TestAnEscapedCharacterIsStillACharacter(t *testing.T) { for _, literal := range []string{`'\n'`, `'\''`, `'\u{1F600}'`} { src := "let c = " + literal + "; let n = 1;" t.Run(literal, func(t *testing.T) { if got := classOfFirst(t, src, literal); got != syntax.ClassChar { t.Errorf("%s is %v, want char", literal, got) } if got := classOfFirst(t, src, "let n"); got != syntax.ClassKeyword { t.Errorf("the code after %s is %v, want keyword", literal, got) } }) } } func TestNumbersTakeTheirSeparatorsPrefixesAndSuffixes(t *testing.T) { tests := []string{"1_000", "0xFF", "0b1010", "0o77", "1.5e-3", "42u8", "3.0f64"} for _, literal := range tests { t.Run(literal, func(t *testing.T) { src := "let n = " + literal + ";" spans := Highlight(src) start := strings.Index(src, literal) for col := start; col < start+len(literal); col++ { class, ok := classAt(spans, 0, col) if !ok || class != syntax.ClassNumber { t.Fatalf("column %d of %q is %v (covered: %v), want the whole literal to be a number", col-start, literal, class, ok) } } }) } } func TestARangeIsNotADecimalPoint(t *testing.T) { // `0..10` is two numbers and a range operator, not one strange number. const src = "for i in 0..10 {}" spans := Highlight(src) class, ok := classAt(spans, 0, strings.Index(src, "..")) if !ok || class != syntax.ClassOperator { t.Errorf("the .. of a range is %v (covered: %v), want operator", class, ok) } } func TestSpansNeverStraddleALineBreak(t *testing.T) { spans := Highlight("/* a\nb */\nfn main() {}") for line, onLine := range spans { for _, span := range onLine { if span.Start < 0 || span.End < span.Start { t.Errorf("line %d holds a nonsense span %+v", line, span) } } } } func TestSpansOnALineAreOrderedAndDoNotOverlap(t *testing.T) { // The editor draws them in order and assumes they do not overlap. const src = `fn f(x: &'a str) -> Option { Some(b"hi"[0]) }` for line, onLine := range Highlight(src) { previousEnd := 0 for _, span := range onLine { if span.Start < previousEnd { t.Errorf("line %d: span %+v starts before the previous one ended at %d", line, span, previousEnd) } previousEnd = span.End } } } func TestBrokenSourceIsStillColoured(t *testing.T) { // Source under the cursor is invalid most of the time it is being typed. tests := []string{ `let s = "unterminated`, "fn f( {", "let x = 'unterminated", "#[derive(", "r#\"unterminated", } for _, src := range tests { t.Run(src, func(t *testing.T) { spans := Highlight(src) if len(spans) != 1 { t.Fatalf("Highlight(%q) returned %d lines, want 1", src, len(spans)) } }) } } func TestColumnsAreCountedInRunesNotBytes(t *testing.T) { // A byte offset would put the spans of a line with an accent in it out of // step with what is drawn. const src = `let café = "thé";` spans := Highlight(src) // "thé" starts at rune column 12: l-e-t-space-c-a-f-é-space-=-space-" class, ok := classAt(spans, 0, 12) if !ok || class != syntax.ClassString { t.Errorf("the string after an accented identifier is %v (covered: %v), want string", class, ok) } } func TestAWholeFileOfRustColoursWithoutPanicking(t *testing.T) { // A broad sweep over the constructs the scanner knows, run for its own // sake: the classes are checked one at a time above. const src = `//! A module doc comment. use std::collections::HashMap; /// Adds two numbers. #[derive(Debug, Clone)] pub struct Adder<'a> { name: &'a str, seen: HashMap, } impl<'a> Adder<'a> { pub fn new(name: &'a str) -> Self { Self { name, seen: HashMap::new() } } pub fn add(&mut self, a: i64, b: i64) -> Result { let sql = r#"insert into "log" values (?)"#; println!("{sql} {} {}", a, b); match a.checked_add(b) { Some(v) => Ok(v), None => Err(format!("overflow: {a} + {b}")), } } } #[cfg(test)] mod tests { use super::*; #[test] fn it_adds() { assert_eq!(Adder::new("x").add(1, 2).unwrap(), 3); } } ` spans := Highlight(src) if got, want := len(spans), strings.Count(src, "\n")+1; got != want { t.Fatalf("Highlight() returned %d lines, want %d", got, want) } }