turbo-editors/turbo-moonbitpublic Fork 0
v1.0.2
Commits
Clone
git clone https://git.rickub.com/turbo-editors/turbo-moonbit.git
git clone ssh://git@rickub.com/turbo-editors/turbo-moonbit.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

scan_test.go · 468 lines · 16.9 KBGo Blame HistoryRaw
📦 Turbo MoonBit cc1f595 k33g 11h ago1package moonbitlang_test
2
3import (
4 "strings"
5 "testing"
6
7 "rickub.com/turbo-editors/turbo-core/syntax"
8
9 "rickub.com/turbo-editors/turbo-moonbit/internal/moonbitlang"
10)
11
12// coloured is one span with the text it covers, which is what a test wants to
13// talk about: "the word fn is a keyword", not "columns 0 to 2 are class 1".
14type coloured struct {
15 text string
16 class syntax.Class
17}
18
19func (c coloured) String() string { return c.text + ":" + c.class.String() }
20
21// colouredLine returns every span of one line of source, with its text.
22func colouredLine(t *testing.T, src string) []coloured {
23 t.Helper()
24
25 lines := moonbitlang.Highlight(src)
26 if len(lines) != 1 {
27 t.Fatalf("Highlight(%q) returned %d lines, want 1", src, len(lines))
28 }
29 return withText([]rune(src), lines[0])
30}
31
32// withText pairs each span with the runes it covers.
33func withText(line []rune, spans []syntax.Span) []coloured {
34 out := make([]coloured, 0, len(spans))
35 for _, span := range spans {
36 out = append(out, coloured{string(line[span.Start:span.End]), span.Class})
37 }
38 return out
39}
40
41// find returns the span covering exactly the given text, if there is one.
42func find(spans []coloured, text string) (coloured, bool) {
43 for _, span := range spans {
44 if span.text == text {
45 return span, true
46 }
47 }
48 return coloured{}, false
49}
50
51// assertClass fails unless one span covers exactly text and has the wanted
52// class. Asking for the whole text means a scanner that split a construct in
53// two is caught, not only one that coloured it wrongly.
54func assertClass(t *testing.T, src, text string, want syntax.Class) {
55 t.Helper()
56
57 spans := colouredLine(t, src)
58 got, ok := find(spans, text)
59 if !ok {
60 t.Fatalf("in %q: no single span covers %q; got %v", src, text, spans)
61 }
62 if got.class != want {
63 t.Errorf("in %q: %q is %s, want %s", src, text, got.class, want)
64 }
65}
66
67// --- the three invariants the editor relies on ------------------------------
68
69// A representative body of MoonBit, used by the invariant tests below. It is
70// deliberately a mixture: every construct the scanner knows, some broken input,
71// and the constructs that most easily run into one another.
72const sample = `///| A doc comment.
73// An ordinary one.
74#deprecated("use area instead")
75pub fn area(shape : Shape, scale~ : Double = 1.0) -> Double raise {
76 let table : Map[String, Int] = { "a": 1, "b": 0xFF }
77 let text = "got \{shape} and \{scale}"
78 let raw =
79 #|literal ${not interpolated}
80 $|and \{interpolated}
81 guard scale > 0.0 else { fail("scale") }
82 match shape {
83 Circle(r) => 3.14159 * r * r
84 Rect(w, h) => w * h
85 }
86 let range = 1..=2
87 let pair = (1, 2).0
88 let bytes = b"\xFF\x00"
89 let ch = 'x'
90 let pattern = re"[a-z]+"
91 ignore(@json.parse(text))
92 let broken = "unterminated
93 let after = 1
94}`
95
96func TestEveryLineGetsExactlyOneEntry(t *testing.T) {
97 // The editor indexes the result by line number without checking, so a
98 // scanner that returned one entry fewer would draw every line below the
99 // gap in the wrong colours.
100 src := sample + "\n\n\ntrailing\n"
101 want := len(strings.Split(src, "\n"))
102
103 if got := len(moonbitlang.Highlight(src)); got != want {
104 t.Errorf("Highlight returned %d lines for %d lines of source", got, want)
105 }
106}
107
108func TestSpansAreInOrderAndDoNotOverlap(t *testing.T) {
109 // Spans are drawn in the order they arrive. Two out of order paint over
110 // each other, and nothing fails.
111 for number, spans := range moonbitlang.Highlight(sample) {
112 line := []rune(strings.Split(sample, "\n")[number])
113 previousEnd := 0
114
115 for _, span := range spans {
116 switch {
117 case span.Start < previousEnd:
118 t.Errorf("line %d: span %v starts before the previous one ended at %d", number+1, span, previousEnd)
119 case span.Start >= span.End:
120 t.Errorf("line %d: span %v is empty or inverted", number+1, span)
121 case span.End > len(line):
122 t.Errorf("line %d: span %v runs past the %d runes of the line", number+1, span, len(line))
123 }
124 previousEnd = span.End
125 }
126 }
127}
128
129func TestBrokenInputStillColours(t *testing.T) {
130 // Source under the cursor is invalid most of the time it is being typed.
131 broken := []string{
132 `let x = "`,
133 `let x = '`,
134 `let x = b"\`,
135 `fn (`,
136 `#`,
137 `#|`,
138 `$|`,
139 `@`,
140 `@/`,
141 `.`,
142 `..`,
143 `1.`,
144 `0x`,
145 `re"`,
146 `}}}`,
147 `let x = 1e`,
148 `let x = 1e-`,
149 `~`,
150 `let x~`,
151 }
152 for _, src := range broken {
153 spans := moonbitlang.Highlight(src)
154 if len(spans) != 1 {
155 t.Errorf("Highlight(%q) returned %d lines, want 1", src, len(spans))
156 }
157 }
158}
159
160func TestNothingCarriesOntoTheNextLine(t *testing.T) {
161 // This is the property that makes MoonBit's scanner stateless: no literal
162 // may reach the next line, so a stray quote must not paint the rest of the
163 // file. Every other editor in this family would carry here.
164 //
165 // Today the carry type is empty, so this cannot fail — and that is why it
166 // is written down. It is the guard on the type: a later change that gives
167 // carry a field has to keep every one of these openers from reaching the
168 // line below, and this is where it finds out that it did not.
169 openers := []string{`"`, `'`, `b"`, `b'`, `re"`, `#|`, `$|`, `#deprecated(`, `//`}
170
171 for _, opener := range openers {
172 src := "let a = " + opener + "\nfn main {\n println(\"hi\")\n}"
173 lines := moonbitlang.Highlight(src)
174
175 second := withText([]rune("fn main {"), lines[1])
176 if got, ok := find(second, "fn"); !ok || got.class != syntax.ClassKeyword {
177 t.Errorf("after a line opening with %q, fn on the next line is %v, want a keyword", opener, second)
178 }
179 }
180}
181
182func TestAnEmptyDocumentIsOneEmptyLine(t *testing.T) {
183 if got := moonbitlang.Highlight(""); len(got) != 1 || len(got[0]) != 0 {
184 t.Errorf("Highlight(\"\") = %v, want one line with no spans", got)
185 }
186}
187
188func TestCRLFColoursTheSameAsLF(t *testing.T) {
189 unix := moonbitlang.Highlight("fn main {\n println(\"hi\")\n}")
190 windows := moonbitlang.Highlight("fn main {\r\n println(\"hi\")\r\n}")
191
192 if len(unix) != len(windows) {
193 t.Fatalf("CRLF gave %d lines, LF gave %d", len(windows), len(unix))
194 }
195 for i := range unix {
196 if len(unix[i]) != len(windows[i]) {
197 t.Errorf("line %d: CRLF gave %v, LF gave %v", i+1, windows[i], unix[i])
198 }
199 }
200}
201
202// --- one case per construct -------------------------------------------------
203
204func TestConstructs(t *testing.T) {
205 cases := []struct {
206 name string
207 src string
208 text string
209 class syntax.Class
210 }{
211 {"line comment", `let x = 1 // why`, `// why`, syntax.ClassComment},
212 {"doc comment", `/// Adds two numbers.`, `/// Adds two numbers.`, syntax.ClassComment},
213 {"section marker", `///|`, `///|`, syntax.ClassComment},
214 {"comment wins over division", `// a / b`, `// a / b`, syntax.ClassComment},
215
216 {"attribute", `#deprecated("use area")`, `#deprecated("use area")`, syntax.ClassAttribute},
217 {"namespaced attribute", `#custom.attribute(key="v")`, `#custom.attribute(key="v")`, syntax.ClassAttribute},
218 {"bare attribute", `#external`, `#external`, syntax.ClassAttribute},
219
220 {"raw multiline prefix", ` #|hello`, `#|`, syntax.ClassPunctuation},
221 {"raw multiline text", ` #|hello`, `hello`, syntax.ClassString},
222 {"interpolated multiline prefix", ` $|hi \{name}`, `$|`, syntax.ClassPunctuation},
223 {"interpolated multiline text", ` $|hi \{name}`, `hi \{name}`, syntax.ClassString},
224
225 {"string", `let s = "hi"`, `"hi"`, syntax.ClassString},
226 {"string stops at its closing quote", `let s = "hi" + name`, `"hi"`, syntax.ClassString},
227 {"code after a string is still code", `let s = "hi" + name`, `name`, syntax.ClassIdentifier},
228 {"char stops at its closing quote", `let c = 'x' + 1`, `'x'`, syntax.ClassChar},
229 {"code after a char is still code", `let c = 'x' + 1`, `1`, syntax.ClassNumber},
230 {"empty string", `let s = ""`, `""`, syntax.ClassString},
231 {"string with interpolation", `let s = "a \{b} c"`, `"a \{b} c"`, syntax.ClassString},
232 {"string with escaped quote", `let s = "a \" b"`, `"a \" b"`, syntax.ClassString},
233 {"bytes literal", `let b = b"\xFF"`, `b"\xFF"`, syntax.ClassString},
234 {"regex literal", `let r = re"[a-z]+"`, `re"[a-z]+"`, syntax.ClassString},
235 {"char literal", `let c = 'x'`, `'x'`, syntax.ClassChar},
236 {"escaped char literal", `let c = '\n'`, `'\n'`, syntax.ClassChar},
237 {"byte literal", `let c = b'x'`, `b'x'`, syntax.ClassChar},
238
239 {"decimal", `let n = 1_000`, `1_000`, syntax.ClassNumber},
240 {"hexadecimal", `let n = 0xFF_FF`, `0xFF_FF`, syntax.ClassNumber},
241 {"octal", `let n = 0o17`, `0o17`, syntax.ClassNumber},
242 {"binary", `let n = 0b1010`, `0b1010`, syntax.ClassNumber},
243 {"double", `let n = 1.5`, `1.5`, syntax.ClassNumber},
244 {"double with a trailing point", `let n = 1.`, `1.`, syntax.ClassNumber},
245 {"exponent", `let n = 1.5e-3`, `1.5e-3`, syntax.ClassNumber},
246 {"hex float", `let n = 0x1.8p3F`, `0x1.8p3F`, syntax.ClassNumber},
247 {"uint suffix", `let n = 42U`, `42U`, syntax.ClassNumber},
248 {"uint64 suffix", `let n = 42UL`, `42UL`, syntax.ClassNumber},
249 {"bigint suffix", `let n = 42N`, `42N`, syntax.ClassNumber},
250 {"float suffix", `let n = 1.0F`, `1.0F`, syntax.ClassNumber},
251
252 {"keyword", `pub fn area() -> Int {`, `fn`, syntax.ClassKeyword},
253 {"visibility keyword", `pub fn area() -> Int {`, `pub`, syntax.ClassKeyword},
254 {"try with a bang", `try! risky()`, `try!`, syntax.ClassKeyword},
255 {"guard with a bang", `guard! x`, `guard!`, syntax.ClassKeyword},
256 {"constant", `let ok = true`, `true`, syntax.ClassConstant},
257 {"option constructor", `Some(1)`, `Some`, syntax.ClassConstant},
258 {"result constructor", `Err("no")`, `Err`, syntax.ClassConstant},
259 {"builtin", `println("hi")`, `println`, syntax.ClassBuiltin},
260 {"builtin in a test", `assert_eq(1, 1)`, `assert_eq`, syntax.ClassBuiltin},
261
262 {"built-in type", `let n : Int = 1`, `Int`, syntax.ClassType},
263 {"generic type", `let m : Map[String, Int] = {}`, `Map`, syntax.ClassType},
264 {"a type nobody built in", `let p : Point = origin`, `Point`, syntax.ClassType},
265 {"a constructor of your own", `Circle(1.0)`, `Circle`, syntax.ClassType},
266 {"call", `area(shape)`, `area`, syntax.ClassFunction},
267 {"plain identifier", `let shape = other`, `other`, syntax.ClassIdentifier},
268
269 {"package name", `@json.parse(text)`, `@json`, syntax.ClassType},
270 {"nested package name", `@moonbitlang/core/builtin.foo()`, `@moonbitlang/core/builtin`, syntax.ClassType},
271 {"hyphenated package name", `@my-pkg.foo()`, `@my-pkg`, syntax.ClassType},
272
273 {"label", `fn greet(name~ : String)`, `name~`, syntax.ClassAttribute},
274 {"optional label", `fn greet(name~ : String = "x")`, `name~`, syntax.ClassAttribute},
275
276 {"method call", `xs.length()`, `length`, syntax.ClassFunction},
277 {"field access", `point.x`, `x`, syntax.ClassIdentifier},
278 {"tuple accessor dot", `pair.0`, `.`, syntax.ClassPunctuation},
279 {"tuple accessor index", `pair.0`, `0`, syntax.ClassNumber},
280
281 {"range operator", `for i in 1..=10 {`, `..`, syntax.ClassOperator},
282 {"pipe operator", `x |> f`, `|>`, syntax.ClassOperator},
283 {"arrow", `Circle(r) => r`, `=>`, syntax.ClassOperator},
284 {"colon is an operator rune", `Type::method`, `::`, syntax.ClassOperator},
285 {"brace", `fn main {`, `{`, syntax.ClassPunctuation},
286 }
287
288 for _, c := range cases {
289 t.Run(c.name, func(t *testing.T) {
290 assertClass(t, c.src, c.text, c.class)
291 })
292 }
293}
294
295func TestRangeStopsTheNumberBeforeIt(t *testing.T) {
296 // "Before .., an integer ends first, so 1..=2 begins with 1 and ..=". A
297 // scanner that swallowed any dot would read 1. as a double and miscolour
298 // every range in the file.
299 spans := colouredLine(t, `for i in 1..=10 {`)
300
301 if got, ok := find(spans, "1"); !ok || got.class != syntax.ClassNumber {
302 t.Errorf("in 1..=10, the 1 is %v, want a number on its own; got %v", got, spans)
303 }
304 if _, ok := find(spans, "1."); ok {
305 t.Errorf("in 1..=10, the scanner read 1. as a double: %v", spans)
306 }
307}
308
309// --- one case per thing the scanner deliberately refuses --------------------
310
311func TestRefusals(t *testing.T) {
312 cases := []struct {
313 name string
314 why string
315 src string
316 text string
317 class syntax.Class
318 }{
319 {
320 name: "an interpolated expression is not code",
321 why: "finding where one ends needs the parser; a brace counter that got it wrong would end the string early",
322 src: `let s = "\{count + 1}"`,
323 text: `"\{count + 1}"`,
324 class: syntax.ClassString,
325 },
326 {
327 name: "a lower-case number suffix is not a suffix",
328 why: "the grammar says the suffixes are upper case, so 42u is 42 and then the name u",
329 src: `let n = 42u`,
330 text: `42`,
331 class: syntax.ClassNumber,
332 },
333 {
334 name: "a leading dot is never a number",
335 why: "MoonBit requires a digit before the point, so .5 is a dot and then a name",
336 src: `let n = .5`,
337 text: `.`,
338 class: syntax.ClassPunctuation,
339 },
340 {
341 name: "a keyword after a dot is a field name",
342 why: "dot-identifiers use the identifier case rules without consulting the keyword table, so .if is valid",
343 src: `config.if`,
344 text: `if`,
345 class: syntax.ClassIdentifier,
346 },
347 {
348 name: "a reserved word is not a keyword",
349 why: "move, ref and the rest are identifiers the compiler warns about; colouring them would deny a valid name",
350 src: `let ref = 1`,
351 text: `ref`,
352 class: syntax.ClassIdentifier,
353 },
354 {
355 name: "an enum constructor of your own is a type",
356 why: "nothing in the syntax separates Circle(1.0) from a type applied to arguments",
357 src: `Circle(1.0)`,
358 text: `Circle`,
359 class: syntax.ClassType,
360 },
361 {
362 name: "an unterminated literal stops at the line",
363 why: "a newline before the closing quote is an unterminated-literal error, so there is nothing to carry",
364 src: `let s = "oops`,
365 text: `"oops`,
366 class: syntax.ClassString,
367 },
368 {
369 name: "an upper-case name cannot form a label",
370 why: "the grammar says ASCII-uppercase identifiers and keywords cannot form labels",
371 src: `Foo~`,
372 text: `Foo`,
373 class: syntax.ClassType,
374 },
375 {
376 name: "a keyword cannot form a label",
377 why: "same rule; let~ is not a labelled argument called let",
378 src: `let~`,
379 text: `let`,
380 class: syntax.ClassKeyword,
381 },
382 {
383 name: "b is only a prefix when the quote touches it",
384 why: "otherwise the variable b in `b + 1` would open a literal",
385 src: `b + 1`,
386 text: `b`,
387 class: syntax.ClassIdentifier,
388 },
389 {
390 name: "a package part must follow the slash",
391 why: "@a/2 is a package and then a division, not a package part called 2",
392 src: `@a/2`,
393 text: `@a`,
394 class: syntax.ClassType,
395 },
396 {
397 name: "a doc comment is coloured like any other comment",
398 why: "turbo-core's Class set is closed and has one comment class, which is what lets one theme colour every language",
399 src: `/// docs`,
400 text: `/// docs`,
401 class: syntax.ClassComment,
402 },
403 }
404
405 for _, c := range cases {
406 t.Run(c.name, func(t *testing.T) {
407 assertClass(t, c.src, c.text, c.class)
408 })
409 }
410}
411
412// A string *nested inside* an interpolation ends the outer literal, because
413// the scanner takes the first unescaped quote as the closer. The grammar says
414// otherwise — "braces inside nested literals do not affect matching" — so this
415// is the precise shape of what the scanner gives up by not parsing, and it is
416// pinned here rather than described loosely.
417//
418// The spans stay in order and never overlap, so nothing downstream breaks; the
419// cost is that text inside the nested literal may take a different colour.
420func TestAStringInsideAnInterpolationEndsTheOuterLiteral(t *testing.T) {
421 // The ordinary case is one span, which is what almost every interpolation
422 // in real MoonBit looks like.
423 if spans := colouredLine(t, `let s = "a \{b} c"`); len(spans) != 4 {
424 t.Errorf(`"a \{b} c" gave %v, want the whole literal as one span`, spans)
425 }
426
427 // With a nested literal it is not one span, and the inside of that literal
428 // is not coloured as a string.
429 spans := colouredLine(t, `let s = "a \{f("x")} c"`)
430 if got, ok := find(spans, "x"); !ok || got.class != syntax.ClassIdentifier {
431 t.Errorf(`"a \{f("x")} c" coloured the nested literal's contents as %v, want the documented identifier`, spans)
432 }
433
434 // Whatever it does colour, the invariants hold.
435 previousEnd := 0
436 for _, span := range moonbitlang.Highlight(`let s = "a \{f("x")} c"`)[0] {
437 if span.Start < previousEnd {
438 t.Errorf("spans overlap: %v", spans)
439 }
440 previousEnd = span.End
441 }
442}
443
444func TestANonASCIIIdentifierIsLeftUncoloured(t *testing.T) {
445 // MoonBit allows CJK and other ranges in identifiers; turbo-core's rune
446 // predicates are ASCII. Such a name is stepped over rather than guessed at,
447 // which is a boundary worth knowing rather than a defect to hide.
448 spans := colouredLine(t, `let 名前 = 1`)
449
450 if _, ok := find(spans, "名前"); ok {
451 t.Errorf("a CJK identifier was coloured: %v", spans)
452 }
453 if got, ok := find(spans, "let"); !ok || got.class != syntax.ClassKeyword {
454 t.Errorf("the rest of the line stopped colouring: %v", spans)
455 }
456}
457
458func TestHighlightIsWhatTheRegistryUses(t *testing.T) {
459 moonbitlang.Register()
460
461 spans := syntax.Highlight(moonbitlang.Language, "fn main {\n")
462 if len(spans) == 0 || len(spans[0]) == 0 {
463 t.Fatalf("syntax.Highlight gave nothing for MoonBit: %v", spans)
464 }
465 if spans[0][0].Class != syntax.ClassKeyword {
466 t.Errorf("the registered highlighter coloured fn as %s, want a keyword", spans[0][0].Class)
467 }
468}