package jslang import ( "strings" "testing" "rickub.com/turbo-editors/turbo-core/syntax" ) // classAt returns the class covering a rune column on a line, and whether any // span covers it at all. It is how nearly every test below asks its question. func classAt(spans [][]syntax.Span, line, col int) (syntax.Class, bool) { if line < 0 || line >= len(spans) { return 0, false } for _, s := range spans[line] { if col >= s.Start && col < s.End { return s.Class, true } } return 0, false } // classOfFirst returns the class of the first occurrence of word in src, as // coloured by a highlighter. func classOfFirst(t *testing.T, highlight func(string) [][]syntax.Span, src, word string) syntax.Class { t.Helper() index := strings.Index(src, word) if index < 0 { t.Fatalf("%q does not appear in the source", word) } line := strings.Count(src[:index], "\n") col := len([]rune(src[strings.LastIndex(src[:index], "\n")+1 : index])) class, ok := classAt(highlight(src), line, col) if !ok { t.Fatalf("no span covers %q at line %d column %d", word, line, col) } return class } // jsClassOf is classOfFirst for JavaScript. func jsClassOf(t *testing.T, src, word string) syntax.Class { t.Helper() return classOfFirst(t, Highlight, src, word) } // wholeWordIs fails unless every column of word, at its first occurrence on // line 0 of src, carries the class. func wholeWordIs(t *testing.T, highlight func(string) [][]syntax.Span, src, word string, want syntax.Class) { t.Helper() spans := highlight(src) start := len([]rune(src[:strings.Index(src, word)])) for col := start; col < start+len([]rune(word)); col++ { class, ok := classAt(spans, 0, col) if !ok || class != want { t.Fatalf("column %d of %q is %v (covered: %v), want the whole of it to be %v", col-start, word, class, ok, want) } } } func TestHighlightReturnsOneEntryPerLine(t *testing.T) { // The editor indexes the result by line number without checking, so a short // result is an index out of range in the middle of a redraw. tests := []struct { name string src string want int }{ {"empty", "", 1}, {"one line without a terminator", "let x = 1;", 1}, {"one line with a terminator", "let x = 1;\n", 2}, {"three lines", "a\nb\nc", 3}, {"a template across lines", "const t = `a\nb\nc`;", 3}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { if got := len(Highlight(tc.src)); got != tc.want { t.Errorf("Highlight(%q) returned %d lines, want %d", tc.src, got, tc.want) } }) } } func TestEachTokenClass(t *testing.T) { const src = `import { readFile } from "node:fs/promises"; // a comment /** a doc comment */ class Greeter { #name; constructor(name) { this.#name = name; } } async function main() { const count = 42; const ratio = 1.5; const text = 'hello'; const template = ` + "`hi ${text}`" + `; const re = /wor+ld/gi; console.log(text, count, ratio, template, re, undefined, null); return await readFile(process.argv[2]); } ` tests := []struct { word string want syntax.Class }{ {"import", syntax.ClassKeyword}, {"from", syntax.ClassKeyword}, {"class", syntax.ClassKeyword}, {"async", syntax.ClassKeyword}, {"function", syntax.ClassKeyword}, {"const count", syntax.ClassKeyword}, {"return", syntax.ClassKeyword}, {"await", syntax.ClassKeyword}, {"// a comment", syntax.ClassComment}, {"/** a doc comment */", syntax.ClassComment}, {"Greeter", syntax.ClassType}, {"#name", syntax.ClassIdentifier}, {"constructor", syntax.ClassFunction}, {"main", syntax.ClassFunction}, {"this", syntax.ClassConstant}, {"undefined", syntax.ClassConstant}, {"null", syntax.ClassConstant}, {`"node:fs/promises"`, syntax.ClassString}, {"'hello'", syntax.ClassString}, {"`hi ${text}`", syntax.ClassString}, {"/wor+ld/gi", syntax.ClassChar}, {"42", syntax.ClassNumber}, {"1.5", syntax.ClassNumber}, {"console", syntax.ClassBuiltin}, {"process", syntax.ClassBuiltin}, {"log", syntax.ClassFunction}, {"readFile", syntax.ClassIdentifier}, {"=", syntax.ClassOperator}, {";", syntax.ClassPunctuation}, } for _, test := range tests { t.Run(test.word, func(t *testing.T) { if got := jsClassOf(t, src, test.word); got != test.want { t.Errorf("%q is %v, want %v", test.word, got, test.want) } }) } } // --- words ------------------------------------------------------------------ func TestTheNameAfterFunctionIsAFunction(t *testing.T) { // By position, whatever the spelling: a generator's * sits between the // keyword and the name, and async in front changes nothing. tests := []struct{ src, name string }{ {"function parse(input) {}", "parse"}, {"function* generate() {}", "generate"}, {"async function fetchAll() {}", "fetchAll"}, {"const f = function named() {};", "named"}, } for _, test := range tests { t.Run(test.src, func(t *testing.T) { if got := jsClassOf(t, test.src, test.name); got != syntax.ClassFunction { t.Errorf("%s is %v, want function", test.name, got) } }) } } func TestTheNameAfterClassIsAType(t *testing.T) { // A class named in lower case is still a class, by position rather than // by spelling. if got := jsClassOf(t, "class widget extends Base {}", "widget"); got != syntax.ClassType { t.Errorf("widget is %v, want type", got) } if got := jsClassOf(t, "class widget extends Base {}", "Base"); got != syntax.ClassType { t.Errorf("Base is %v, want type", got) } } func TestACallIsAFunction(t *testing.T) { if got := jsClassOf(t, "const n = compute(3);", "compute"); got != syntax.ClassFunction { t.Errorf("compute is %v, want function", got) } } func TestAnUpperCaseNameIsAType(t *testing.T) { // Every JavaScript style guide capitalises a class and nothing else, so // a leading capital says class more reliably than any look at the // neighbouring tokens would — including in front of the parenthesis // that would otherwise make it a call. for _, word := range []string{"EventEmitter", "MyError", "Component"} { src := "const x = new " + word + "();" if got := jsClassOf(t, src, word); got != syntax.ClassType { t.Errorf("%s is %v, want type", word, got) } } } func TestALowerCaseNameIsAPlainIdentifier(t *testing.T) { if got := jsClassOf(t, "const total = subtotal + tax;", "subtotal"); got != syntax.ClassIdentifier { t.Errorf("subtotal is %v, want identifier", got) } } func TestDollarAndUnderscoreBeginNames(t *testing.T) { const src = "const $el = _.get(obj, 'a');" if got := jsClassOf(t, src, "$el"); got != syntax.ClassIdentifier { t.Errorf("$el is %v, want identifier", got) } if got := jsClassOf(t, src, "_"); got != syntax.ClassIdentifier { t.Errorf("_ is %v, want identifier", got) } } func TestUnicodeNamesAreNames(t *testing.T) { const src = "const café = 名前 + 1;" if got := jsClassOf(t, src, "café"); got != syntax.ClassIdentifier { t.Errorf("café is %v, want identifier", got) } if got := jsClassOf(t, src, "名前"); got != syntax.ClassIdentifier { t.Errorf("名前 is %v, want identifier", got) } if got := jsClassOf(t, src, "1"); got != syntax.ClassNumber { t.Errorf("the number after a Unicode name is %v, want number", got) } } func TestTheLiteralsAreConstants(t *testing.T) { for _, word := range []string{"true", "false", "null", "undefined", "NaN", "Infinity", "this"} { src := "let v = " + word + ";" if got := jsClassOf(t, src, word); got != syntax.ClassConstant { t.Errorf("%s is %v, want constant", word, got) } } } func TestNodesGlobalsAreBuiltIn(t *testing.T) { // This is a Node editor: process, Buffer and the CommonJS five are as // much part of the language its user writes as Array and Promise. for _, word := range []string{"process", "Buffer", "require", "module", "exports", "__dirname", "console", "setTimeout", "fetch", "Promise", "JSON"} { src := "x = " + word + ";" if got := jsClassOf(t, src, word); got != syntax.ClassBuiltin { t.Errorf("%s is %v, want builtin", word, got) } } } func TestAWordAfterADotIsAPropertyWhateverItIsSpeltLike(t *testing.T) { // map.get(k) is a call, obj.default is a field, and neither is the // keyword it would be on its own. Optional chaining is a member access // too. tests := []struct { src, word string want syntax.Class }{ {"const v = map.get(key);", "get", syntax.ClassFunction}, {"const d = options.default;", "default", syntax.ClassIdentifier}, {"promise.catch(handle);", "catch", syntax.ClassFunction}, {"const c = user?.class;", "class", syntax.ClassIdentifier}, {"const p = obj.process;", "process", syntax.ClassIdentifier}, {"const t = obj.true;", "true", syntax.ClassIdentifier}, } for _, test := range tests { t.Run(test.src, func(t *testing.T) { if got := jsClassOf(t, test.src, test.word); got != test.want { t.Errorf("%s is %v, want %v", test.word, got, test.want) } }) } } func TestAContextualKeywordIsAKeywordOnlyInPosition(t *testing.T) { // get before a method's name is a keyword; get(k) with nothing in front // of it is a function called get. if got := jsClassOf(t, "class C { get name() { return 1; } }", "get"); got != syntax.ClassKeyword { t.Errorf("get before a getter's name is %v, want keyword", got) } if got := jsClassOf(t, "const v = get(key);", "get"); got != syntax.ClassFunction { t.Errorf("get(key) is %v, want function", got) } if got := jsClassOf(t, "for (const x of xs) {}", "of"); got != syntax.ClassKeyword { t.Errorf("of in a for-of is %v, want keyword", got) } if got := jsClassOf(t, "class C { static create() {} }", "static"); got != syntax.ClassKeyword { t.Errorf("static before a member is %v, want keyword", got) } } func TestAsyncIsAKeywordEvenBeforeAParenthesis(t *testing.T) { // async (x) => … is an arrow function, and async there is not a call. if got := jsClassOf(t, "const f = async (x) => x;", "async"); got != syntax.ClassKeyword { t.Errorf("async before an arrow's parameters is %v, want keyword", got) } } func TestAPrivateMemberTakesItsHash(t *testing.T) { // #count is one name. Colouring the # alone would make it read as the // comment marker it is in half the other languages this editor colours. wholeWordIs(t, Highlight, "this.#count += 1;", "#count", syntax.ClassIdentifier) wholeWordIs(t, Highlight, "this.#reset();", "#reset", syntax.ClassFunction) } func TestADecoratorIsAnAttribute(t *testing.T) { wholeWordIs(t, Highlight, "@observable.ref count = 0;", "@observable.ref", syntax.ClassAttribute) if got := jsClassOf(t, "@observable.ref count = 0;", "count"); got != syntax.ClassIdentifier { t.Errorf("the name after a decorator is %v, want identifier", got) } } // --- regular expressions and division ---------------------------------------- func TestARegularExpressionIsColouredAsOneWhereOneMayBegin(t *testing.T) { // After a keyword, an operator, an opening bracket or a comma, a slash // opens a regular expression. The literal is coloured as a character // literal — the class JavaScript has no other use for — flags included. tests := []struct{ src, literal string }{ {"return /wor+ld/gi.test(s);", "/wor+ld/gi"}, {"if (/^#/.test(line)) {}", "/^#/"}, {"const re = /a[/]b\\/c/;", "/a[/]b\\/c/"}, {"lines.filter(/x/.test, re);", "/x/"}, {"x = y ? /a/ : /b/;", "/a/"}, {"/^\\s*$/.test(s);", "/^\\s*$/"}, } for _, test := range tests { t.Run(test.src, func(t *testing.T) { wholeWordIs(t, Highlight, test.src, test.literal, syntax.ClassChar) }) } } func TestASlashAfterAValueDivides(t *testing.T) { // After a name, a number, a string, a closing parenthesis or bracket, a // slash is division, and the thing after it is code. tests := []struct{ src, after string }{ {"const half = total / 2 / count;", "count"}, {"const r = (a + b) / c;", "c"}, {"const r = items[0] / items.length;", "items.length"}, {"const r = 10 / n;", "n"}, {"const r = this / 2;", "2"}, } for _, test := range tests { t.Run(test.src, func(t *testing.T) { if got := jsClassOf(t, test.src, "/"); got != syntax.ClassOperator { t.Errorf("the slash is %v, want operator", got) } word := strings.SplitN(test.after, ".", 2)[0] if got := jsClassOf(t, test.src, word); got == syntax.ClassChar || got == syntax.ClassString { t.Errorf("%s after the slash is %v; the division was read as a regular expression", word, got) } }) } } func TestASlashThatNothingClosesOnItsLineDivides(t *testing.T) { // A regular expression cannot cross a line, so a slash with no closing // slash on its line cannot open one — whatever came before it. That bound // is what keeps a wrong guess to one line. const src = "x = a /\n b;" spans := Highlight(src) class, ok := classAt(spans, 0, strings.Index(src, "/")) if !ok || class != syntax.ClassOperator { t.Errorf("the trailing slash is %v (covered: %v), want operator", class, ok) } class, ok = classAt(spans, 1, 2) if !ok || class != syntax.ClassIdentifier { t.Errorf("the next line is %v (covered: %v), want code", class, ok) } } func TestARegularExpressionMayOpenALine(t *testing.T) { // A statement beginning with a regular expression is more likely than a // line beginning with a division, so the start of a line allows one. wholeWordIs(t, Highlight, "/foo/.test(x) && go();", "/foo/", syntax.ClassChar) } func TestACommentIsNotARegularExpression(t *testing.T) { // // and /* come first, so a comment after a keyword stays a comment. if got := jsClassOf(t, "return // done\n", "// done"); got != syntax.ClassComment { t.Errorf("a comment after return is %v, want comment", got) } if got := jsClassOf(t, "return /* nothing */ x;", "/* nothing */"); got != syntax.ClassComment { t.Errorf("a block comment after return is %v, want comment", got) } } // --- strings and templates --------------------------------------------------- func TestAnEscapedQuoteDoesNotEndAString(t *testing.T) { const src = `let s = "a \" b"; let n = 1;` if got := jsClassOf(t, src, "let n"); got != syntax.ClassKeyword { t.Errorf("the code after an escaped quote is %v, want keyword", got) } } func TestAnUnterminatedStringStopsAtItsLine(t *testing.T) { // JavaScript strings do not cross lines, so the next line is code again // whatever the previous one left open. spans := Highlight("let s = 'unterminated\nlet n = 1;") class, ok := classAt(spans, 1, 0) if !ok || class != syntax.ClassKeyword { t.Errorf("the line after an unterminated string is %v (covered: %v), want code", class, ok) } } func TestATemplateLiteralCarriesAcrossLines(t *testing.T) { spans := Highlight("const t = `one\ntwo ${x}\nthree`;\nlet n = 1;") for line := 1; line <= 2; line++ { class, ok := classAt(spans, line, 0) if !ok || class != syntax.ClassString { t.Errorf("line %d of the template is %v (covered: %v), want string", line, class, ok) } } class, ok := classAt(spans, 3, 0) if !ok || class != syntax.ClassKeyword { t.Errorf("the line after the template is %v (covered: %v), want code", class, ok) } } func TestATemplateThatClosesMidLineLeavesCodeAfterIt(t *testing.T) { if got := jsClassOf(t, "const t = `a\nb`; let n = 1;", "let n"); got != syntax.ClassKeyword { t.Errorf("the code after the closing backtick is %v, want keyword", got) } } func TestAnInterpolationIsPartOfTheTemplate(t *testing.T) { // The whole literal is a string, ${…} included: colouring the code inside // means carrying a nesting depth for a construct that is usually one // short expression. The reference says so. wholeWordIs(t, Highlight, "const t = `hello ${name.toUpperCase()}!`;", "`hello ${name.toUpperCase()}!`", syntax.ClassString) } func TestAnEscapedBacktickDoesNotEndATemplate(t *testing.T) { const src = "const t = `a \\` b`; let n = 1;" if got := jsClassOf(t, src, "let n"); got != syntax.ClassKeyword { t.Errorf("the code after an escaped backtick is %v, want keyword", got) } } // --- comments --------------------------------------------------------------- func TestABlockCommentCarriesAcrossLines(t *testing.T) { spans := Highlight("/* one\ntwo\n*/ let x = 1;") for line := range 2 { class, ok := classAt(spans, line, 0) if !ok || class != syntax.ClassComment { t.Errorf("line %d is %v (covered: %v), want comment", line, class, ok) } } class, ok := classAt(spans, 2, 3) // the "l" of let if !ok || class != syntax.ClassKeyword { t.Errorf("the code after the comment is %v (covered: %v), want keyword", class, ok) } } func TestBlockCommentsDoNotNest(t *testing.T) { // JavaScript ends a block comment at the first */, whatever opened inside // it. A depth here would be a claim about a different language. const src = "/* a /* b */ let x = 1;" if got := jsClassOf(t, src, "let"); got != syntax.ClassKeyword { t.Errorf("the code after the first */ is %v, want keyword", got) } } func TestAHashbangIsACommentOnTheFirstLineOnly(t *testing.T) { spans := Highlight("#!/usr/bin/env node\nconst x = 1;\n#!not a hashbang") class, ok := classAt(spans, 0, 0) if !ok || class != syntax.ClassComment { t.Errorf("the hashbang is %v (covered: %v), want comment", class, ok) } class, ok = classAt(spans, 1, 0) if !ok || class != syntax.ClassKeyword { t.Errorf("the line after the hashbang is %v (covered: %v), want code", class, ok) } if class, _ := classAt(spans, 2, 0); class == syntax.ClassComment { t.Error("#! on a later line was read as a hashbang") } } // --- numbers ---------------------------------------------------------------- func TestNumbersTakeTheirSeparatorsPrefixesAndSuffixes(t *testing.T) { tests := []string{"1_000", "0xFF", "0o17", "0b1010", "1.5e-3", "2E+10", ".5", "10n", "0xFFn"} for _, literal := range tests { t.Run(literal, func(t *testing.T) { wholeWordIs(t, Highlight, "const n = "+literal+";", literal, syntax.ClassNumber) }) } } func TestAPlusAfterAHexadecimalLiteralIsASum(t *testing.T) { // 0xE ends in an E, and the + straight after it is not an exponent's // sign — written without spaces, because a space already ends the // number and would hide a scanner that got this wrong. const src = "const n = 0xE+1;" if got := jsClassOf(t, src, "+"); got != syntax.ClassOperator { t.Errorf("the + after 0xE is %v, want operator", got) } if got := jsClassOf(t, src, "1;"); got != syntax.ClassNumber { t.Errorf("the 1 after the + is %v, want number", got) } } func TestANumbersDotIsOnlyADotWhenADigitFollows(t *testing.T) { // 1.toString() is invalid, but it is typed; the 1 stops at the dot and // the method after it is a method. const src = "1.toString();" if got := jsClassOf(t, src, "toString"); got != syntax.ClassFunction { t.Errorf("toString after 1. is %v, want function", got) } } // --- punctuation and operators ------------------------------------------------ func TestASpreadIsOneThing(t *testing.T) { const src = "const all = [...first, ...rest];" wholeWordIs(t, Highlight, src, "...", syntax.ClassPunctuation) if got := jsClassOf(t, src, "first"); got != syntax.ClassIdentifier { t.Errorf("the name after a spread is %v, want identifier", got) } } func TestOptionalChainingAndArrowsAreOperators(t *testing.T) { const src = "const f = (x) => x?.name ?? 'none';" wholeWordIs(t, Highlight, src, "=>", syntax.ClassOperator) wholeWordIs(t, Highlight, src, "?.", syntax.ClassOperator) wholeWordIs(t, Highlight, src, "??", syntax.ClassOperator) } // --- the contracts the editor relies on -------------------------------------- func TestSpansOnALineAreOrderedAndDoNotOverlap(t *testing.T) { // The editor draws them in order and assumes they do not overlap. const src = "class A extends B { #x = /re/g; static of() { return this.#x / 2 ?? `${a}`; } }" for line, onLine := range Highlight(src) { previousEnd := 0 for _, span := range onLine { if span.Start < previousEnd { t.Errorf("line %d: span %+v starts before the previous one ended at %d", line, span, previousEnd) } if span.End <= span.Start { t.Errorf("line %d: span %+v is empty or backwards", line, span) } previousEnd = span.End } } } func TestBrokenSourceIsStillColoured(t *testing.T) { // Source under the cursor is invalid most of the time it is being typed. tests := []string{ `let s = "unterminated`, "function f( {", "const re = /unterminated", "class {", "x = `unterminated", "/* unterminated", "@", "#", "...", "const = ;", } for _, src := range tests { t.Run(src, func(t *testing.T) { spans := Highlight(src) if len(spans) != 1 { t.Fatalf("Highlight(%q) returned %d lines, want 1", src, len(spans)) } }) } } func TestColumnsAreCountedInRunesNotBytes(t *testing.T) { // A byte offset would put the spans of a line with an accent in it out of // step with what is drawn. const src = `let café = "thé";` spans := Highlight(src) // "thé" starts at rune column 11: l-e-t-space-c-a-f-é-space-=-space-" class, ok := classAt(spans, 0, 11) if !ok || class != syntax.ClassString { t.Errorf("the string after an accented identifier is %v (covered: %v), want string", class, ok) } } func TestAWholeFileOfJavaScriptColoursWithoutPanicking(t *testing.T) { // A broad sweep over the constructs the scanner knows, run for its own // sake: the classes are checked one at a time above. const src = `#!/usr/bin/env node // A small HTTP server. import { createServer } from "node:http"; import { readFile } from "node:fs/promises"; const PORT = process.env.PORT ?? 8080; const ROUTE = /^\/api\/(\w+)$/; /** * Answers one request. * @param {import("node:http").IncomingMessage} request */ async function handle(request, response) { const match = ROUTE.exec(request.url); if (!match) { response.writeHead(404); return response.end(); } const [, name] = match; const body = await readFile(` + "`./data/${name}.json`" + `, "utf8"); response.writeHead(200, { "content-type": "application/json" }); response.end(body); } class Counter { #count = 0n; static #instances = new Set(); constructor() { Counter.#instances.add(this); } get count() { return this.#count; } increment(by = 1) { this.#count += BigInt(by); return this; } } createServer(handle).listen(PORT, () => { console.log(` + "`listening on ${PORT}`" + `); }); ` spans := Highlight(src) if got, want := len(spans), strings.Count(src, "\n")+1; got != want { t.Fatalf("Highlight() returned %d lines, want %d", got, want) } }