package moonbitlang_test import ( "strings" "testing" "rickub.com/turbo-editors/turbo-core/syntax" "rickub.com/turbo-editors/turbo-moonbit/internal/moonbitlang" ) // coloured is one span with the text it covers, which is what a test wants to // talk about: "the word fn is a keyword", not "columns 0 to 2 are class 1". type coloured struct { text string class syntax.Class } func (c coloured) String() string { return c.text + ":" + c.class.String() } // colouredLine returns every span of one line of source, with its text. func colouredLine(t *testing.T, src string) []coloured { t.Helper() lines := moonbitlang.Highlight(src) if len(lines) != 1 { t.Fatalf("Highlight(%q) returned %d lines, want 1", src, len(lines)) } return withText([]rune(src), lines[0]) } // withText pairs each span with the runes it covers. func withText(line []rune, spans []syntax.Span) []coloured { out := make([]coloured, 0, len(spans)) for _, span := range spans { out = append(out, coloured{string(line[span.Start:span.End]), span.Class}) } return out } // find returns the span covering exactly the given text, if there is one. func find(spans []coloured, text string) (coloured, bool) { for _, span := range spans { if span.text == text { return span, true } } return coloured{}, false } // assertClass fails unless one span covers exactly text and has the wanted // class. Asking for the whole text means a scanner that split a construct in // two is caught, not only one that coloured it wrongly. func assertClass(t *testing.T, src, text string, want syntax.Class) { t.Helper() spans := colouredLine(t, src) got, ok := find(spans, text) if !ok { t.Fatalf("in %q: no single span covers %q; got %v", src, text, spans) } if got.class != want { t.Errorf("in %q: %q is %s, want %s", src, text, got.class, want) } } // --- the three invariants the editor relies on ------------------------------ // A representative body of MoonBit, used by the invariant tests below. It is // deliberately a mixture: every construct the scanner knows, some broken input, // and the constructs that most easily run into one another. const sample = `///| A doc comment. // An ordinary one. #deprecated("use area instead") pub fn area(shape : Shape, scale~ : Double = 1.0) -> Double raise { let table : Map[String, Int] = { "a": 1, "b": 0xFF } let text = "got \{shape} and \{scale}" let raw = #|literal ${not interpolated} $|and \{interpolated} guard scale > 0.0 else { fail("scale") } match shape { Circle(r) => 3.14159 * r * r Rect(w, h) => w * h } let range = 1..=2 let pair = (1, 2).0 let bytes = b"\xFF\x00" let ch = 'x' let pattern = re"[a-z]+" ignore(@json.parse(text)) let broken = "unterminated let after = 1 }` func TestEveryLineGetsExactlyOneEntry(t *testing.T) { // The editor indexes the result by line number without checking, so a // scanner that returned one entry fewer would draw every line below the // gap in the wrong colours. src := sample + "\n\n\ntrailing\n" want := len(strings.Split(src, "\n")) if got := len(moonbitlang.Highlight(src)); got != want { t.Errorf("Highlight returned %d lines for %d lines of source", got, want) } } func TestSpansAreInOrderAndDoNotOverlap(t *testing.T) { // Spans are drawn in the order they arrive. Two out of order paint over // each other, and nothing fails. for number, spans := range moonbitlang.Highlight(sample) { line := []rune(strings.Split(sample, "\n")[number]) previousEnd := 0 for _, span := range spans { switch { case span.Start < previousEnd: t.Errorf("line %d: span %v starts before the previous one ended at %d", number+1, span, previousEnd) case span.Start >= span.End: t.Errorf("line %d: span %v is empty or inverted", number+1, span) case span.End > len(line): t.Errorf("line %d: span %v runs past the %d runes of the line", number+1, span, len(line)) } previousEnd = span.End } } } func TestBrokenInputStillColours(t *testing.T) { // Source under the cursor is invalid most of the time it is being typed. broken := []string{ `let x = "`, `let x = '`, `let x = b"\`, `fn (`, `#`, `#|`, `$|`, `@`, `@/`, `.`, `..`, `1.`, `0x`, `re"`, `}}}`, `let x = 1e`, `let x = 1e-`, `~`, `let x~`, } for _, src := range broken { spans := moonbitlang.Highlight(src) if len(spans) != 1 { t.Errorf("Highlight(%q) returned %d lines, want 1", src, len(spans)) } } } func TestNothingCarriesOntoTheNextLine(t *testing.T) { // This is the property that makes MoonBit's scanner stateless: no literal // may reach the next line, so a stray quote must not paint the rest of the // file. Every other editor in this family would carry here. // // Today the carry type is empty, so this cannot fail — and that is why it // is written down. It is the guard on the type: a later change that gives // carry a field has to keep every one of these openers from reaching the // line below, and this is where it finds out that it did not. openers := []string{`"`, `'`, `b"`, `b'`, `re"`, `#|`, `$|`, `#deprecated(`, `//`} for _, opener := range openers { src := "let a = " + opener + "\nfn main {\n println(\"hi\")\n}" lines := moonbitlang.Highlight(src) second := withText([]rune("fn main {"), lines[1]) if got, ok := find(second, "fn"); !ok || got.class != syntax.ClassKeyword { t.Errorf("after a line opening with %q, fn on the next line is %v, want a keyword", opener, second) } } } func TestAnEmptyDocumentIsOneEmptyLine(t *testing.T) { if got := moonbitlang.Highlight(""); len(got) != 1 || len(got[0]) != 0 { t.Errorf("Highlight(\"\") = %v, want one line with no spans", got) } } func TestCRLFColoursTheSameAsLF(t *testing.T) { unix := moonbitlang.Highlight("fn main {\n println(\"hi\")\n}") windows := moonbitlang.Highlight("fn main {\r\n println(\"hi\")\r\n}") if len(unix) != len(windows) { t.Fatalf("CRLF gave %d lines, LF gave %d", len(windows), len(unix)) } for i := range unix { if len(unix[i]) != len(windows[i]) { t.Errorf("line %d: CRLF gave %v, LF gave %v", i+1, windows[i], unix[i]) } } } // --- one case per construct ------------------------------------------------- func TestConstructs(t *testing.T) { cases := []struct { name string src string text string class syntax.Class }{ {"line comment", `let x = 1 // why`, `// why`, syntax.ClassComment}, {"doc comment", `/// Adds two numbers.`, `/// Adds two numbers.`, syntax.ClassComment}, {"section marker", `///|`, `///|`, syntax.ClassComment}, {"comment wins over division", `// a / b`, `// a / b`, syntax.ClassComment}, {"attribute", `#deprecated("use area")`, `#deprecated("use area")`, syntax.ClassAttribute}, {"namespaced attribute", `#custom.attribute(key="v")`, `#custom.attribute(key="v")`, syntax.ClassAttribute}, {"bare attribute", `#external`, `#external`, syntax.ClassAttribute}, {"raw multiline prefix", ` #|hello`, `#|`, syntax.ClassPunctuation}, {"raw multiline text", ` #|hello`, `hello`, syntax.ClassString}, {"interpolated multiline prefix", ` $|hi \{name}`, `$|`, syntax.ClassPunctuation}, {"interpolated multiline text", ` $|hi \{name}`, `hi \{name}`, syntax.ClassString}, {"string", `let s = "hi"`, `"hi"`, syntax.ClassString}, {"string stops at its closing quote", `let s = "hi" + name`, `"hi"`, syntax.ClassString}, {"code after a string is still code", `let s = "hi" + name`, `name`, syntax.ClassIdentifier}, {"char stops at its closing quote", `let c = 'x' + 1`, `'x'`, syntax.ClassChar}, {"code after a char is still code", `let c = 'x' + 1`, `1`, syntax.ClassNumber}, {"empty string", `let s = ""`, `""`, syntax.ClassString}, {"string with interpolation", `let s = "a \{b} c"`, `"a \{b} c"`, syntax.ClassString}, {"string with escaped quote", `let s = "a \" b"`, `"a \" b"`, syntax.ClassString}, {"bytes literal", `let b = b"\xFF"`, `b"\xFF"`, syntax.ClassString}, {"regex literal", `let r = re"[a-z]+"`, `re"[a-z]+"`, syntax.ClassString}, {"char literal", `let c = 'x'`, `'x'`, syntax.ClassChar}, {"escaped char literal", `let c = '\n'`, `'\n'`, syntax.ClassChar}, {"byte literal", `let c = b'x'`, `b'x'`, syntax.ClassChar}, {"decimal", `let n = 1_000`, `1_000`, syntax.ClassNumber}, {"hexadecimal", `let n = 0xFF_FF`, `0xFF_FF`, syntax.ClassNumber}, {"octal", `let n = 0o17`, `0o17`, syntax.ClassNumber}, {"binary", `let n = 0b1010`, `0b1010`, syntax.ClassNumber}, {"double", `let n = 1.5`, `1.5`, syntax.ClassNumber}, {"double with a trailing point", `let n = 1.`, `1.`, syntax.ClassNumber}, {"exponent", `let n = 1.5e-3`, `1.5e-3`, syntax.ClassNumber}, {"hex float", `let n = 0x1.8p3F`, `0x1.8p3F`, syntax.ClassNumber}, {"uint suffix", `let n = 42U`, `42U`, syntax.ClassNumber}, {"uint64 suffix", `let n = 42UL`, `42UL`, syntax.ClassNumber}, {"bigint suffix", `let n = 42N`, `42N`, syntax.ClassNumber}, {"float suffix", `let n = 1.0F`, `1.0F`, syntax.ClassNumber}, {"keyword", `pub fn area() -> Int {`, `fn`, syntax.ClassKeyword}, {"visibility keyword", `pub fn area() -> Int {`, `pub`, syntax.ClassKeyword}, {"try with a bang", `try! risky()`, `try!`, syntax.ClassKeyword}, {"guard with a bang", `guard! x`, `guard!`, syntax.ClassKeyword}, {"constant", `let ok = true`, `true`, syntax.ClassConstant}, {"option constructor", `Some(1)`, `Some`, syntax.ClassConstant}, {"result constructor", `Err("no")`, `Err`, syntax.ClassConstant}, {"builtin", `println("hi")`, `println`, syntax.ClassBuiltin}, {"builtin in a test", `assert_eq(1, 1)`, `assert_eq`, syntax.ClassBuiltin}, {"built-in type", `let n : Int = 1`, `Int`, syntax.ClassType}, {"generic type", `let m : Map[String, Int] = {}`, `Map`, syntax.ClassType}, {"a type nobody built in", `let p : Point = origin`, `Point`, syntax.ClassType}, {"a constructor of your own", `Circle(1.0)`, `Circle`, syntax.ClassType}, {"call", `area(shape)`, `area`, syntax.ClassFunction}, {"plain identifier", `let shape = other`, `other`, syntax.ClassIdentifier}, {"package name", `@json.parse(text)`, `@json`, syntax.ClassType}, {"nested package name", `@moonbitlang/core/builtin.foo()`, `@moonbitlang/core/builtin`, syntax.ClassType}, {"hyphenated package name", `@my-pkg.foo()`, `@my-pkg`, syntax.ClassType}, {"label", `fn greet(name~ : String)`, `name~`, syntax.ClassAttribute}, {"optional label", `fn greet(name~ : String = "x")`, `name~`, syntax.ClassAttribute}, {"method call", `xs.length()`, `length`, syntax.ClassFunction}, {"field access", `point.x`, `x`, syntax.ClassIdentifier}, {"tuple accessor dot", `pair.0`, `.`, syntax.ClassPunctuation}, {"tuple accessor index", `pair.0`, `0`, syntax.ClassNumber}, {"range operator", `for i in 1..=10 {`, `..`, syntax.ClassOperator}, {"pipe operator", `x |> f`, `|>`, syntax.ClassOperator}, {"arrow", `Circle(r) => r`, `=>`, syntax.ClassOperator}, {"colon is an operator rune", `Type::method`, `::`, syntax.ClassOperator}, {"brace", `fn main {`, `{`, syntax.ClassPunctuation}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { assertClass(t, c.src, c.text, c.class) }) } } func TestRangeStopsTheNumberBeforeIt(t *testing.T) { // "Before .., an integer ends first, so 1..=2 begins with 1 and ..=". A // scanner that swallowed any dot would read 1. as a double and miscolour // every range in the file. spans := colouredLine(t, `for i in 1..=10 {`) if got, ok := find(spans, "1"); !ok || got.class != syntax.ClassNumber { t.Errorf("in 1..=10, the 1 is %v, want a number on its own; got %v", got, spans) } if _, ok := find(spans, "1."); ok { t.Errorf("in 1..=10, the scanner read 1. as a double: %v", spans) } } // --- one case per thing the scanner deliberately refuses -------------------- func TestRefusals(t *testing.T) { cases := []struct { name string why string src string text string class syntax.Class }{ { name: "an interpolated expression is not code", why: "finding where one ends needs the parser; a brace counter that got it wrong would end the string early", src: `let s = "\{count + 1}"`, text: `"\{count + 1}"`, class: syntax.ClassString, }, { name: "a lower-case number suffix is not a suffix", why: "the grammar says the suffixes are upper case, so 42u is 42 and then the name u", src: `let n = 42u`, text: `42`, class: syntax.ClassNumber, }, { name: "a leading dot is never a number", why: "MoonBit requires a digit before the point, so .5 is a dot and then a name", src: `let n = .5`, text: `.`, class: syntax.ClassPunctuation, }, { name: "a keyword after a dot is a field name", why: "dot-identifiers use the identifier case rules without consulting the keyword table, so .if is valid", src: `config.if`, text: `if`, class: syntax.ClassIdentifier, }, { name: "a reserved word is not a keyword", why: "move, ref and the rest are identifiers the compiler warns about; colouring them would deny a valid name", src: `let ref = 1`, text: `ref`, class: syntax.ClassIdentifier, }, { name: "an enum constructor of your own is a type", why: "nothing in the syntax separates Circle(1.0) from a type applied to arguments", src: `Circle(1.0)`, text: `Circle`, class: syntax.ClassType, }, { name: "an unterminated literal stops at the line", why: "a newline before the closing quote is an unterminated-literal error, so there is nothing to carry", src: `let s = "oops`, text: `"oops`, class: syntax.ClassString, }, { name: "an upper-case name cannot form a label", why: "the grammar says ASCII-uppercase identifiers and keywords cannot form labels", src: `Foo~`, text: `Foo`, class: syntax.ClassType, }, { name: "a keyword cannot form a label", why: "same rule; let~ is not a labelled argument called let", src: `let~`, text: `let`, class: syntax.ClassKeyword, }, { name: "b is only a prefix when the quote touches it", why: "otherwise the variable b in `b + 1` would open a literal", src: `b + 1`, text: `b`, class: syntax.ClassIdentifier, }, { name: "a package part must follow the slash", why: "@a/2 is a package and then a division, not a package part called 2", src: `@a/2`, text: `@a`, class: syntax.ClassType, }, { name: "a doc comment is coloured like any other comment", why: "turbo-core's Class set is closed and has one comment class, which is what lets one theme colour every language", src: `/// docs`, text: `/// docs`, class: syntax.ClassComment, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { assertClass(t, c.src, c.text, c.class) }) } } // A string *nested inside* an interpolation ends the outer literal, because // the scanner takes the first unescaped quote as the closer. The grammar says // otherwise — "braces inside nested literals do not affect matching" — so this // is the precise shape of what the scanner gives up by not parsing, and it is // pinned here rather than described loosely. // // The spans stay in order and never overlap, so nothing downstream breaks; the // cost is that text inside the nested literal may take a different colour. func TestAStringInsideAnInterpolationEndsTheOuterLiteral(t *testing.T) { // The ordinary case is one span, which is what almost every interpolation // in real MoonBit looks like. if spans := colouredLine(t, `let s = "a \{b} c"`); len(spans) != 4 { t.Errorf(`"a \{b} c" gave %v, want the whole literal as one span`, spans) } // With a nested literal it is not one span, and the inside of that literal // is not coloured as a string. spans := colouredLine(t, `let s = "a \{f("x")} c"`) if got, ok := find(spans, "x"); !ok || got.class != syntax.ClassIdentifier { t.Errorf(`"a \{f("x")} c" coloured the nested literal's contents as %v, want the documented identifier`, spans) } // Whatever it does colour, the invariants hold. previousEnd := 0 for _, span := range moonbitlang.Highlight(`let s = "a \{f("x")} c"`)[0] { if span.Start < previousEnd { t.Errorf("spans overlap: %v", spans) } previousEnd = span.End } } func TestANonASCIIIdentifierIsLeftUncoloured(t *testing.T) { // MoonBit allows CJK and other ranges in identifiers; turbo-core's rune // predicates are ASCII. Such a name is stepped over rather than guessed at, // which is a boundary worth knowing rather than a defect to hide. spans := colouredLine(t, `let 名前 = 1`) if _, ok := find(spans, "名前"); ok { t.Errorf("a CJK identifier was coloured: %v", spans) } if got, ok := find(spans, "let"); !ok || got.class != syntax.ClassKeyword { t.Errorf("the rest of the line stopped colouring: %v", spans) } } func TestHighlightIsWhatTheRegistryUses(t *testing.T) { moonbitlang.Register() spans := syntax.Highlight(moonbitlang.Language, "fn main {\n") if len(spans) == 0 || len(spans[0]) == 0 { t.Fatalf("syntax.Highlight gave nothing for MoonBit: %v", spans) } if spans[0][0].Class != syntax.ClassKeyword { t.Errorf("the registered highlighter coloured fn as %s, want a keyword", spans[0][0].Class) } }