package jslang // Words: what a run of letters turns out to be, and the tables that decide it. import ( "slices" "strings" "unicode" "rickub.com/turbo-editors/turbo-core/syntax" ) // isWordStart reports whether a rune can begin an identifier. JavaScript // identifiers are Unicode: café and 名前 are names, and so is $, which jQuery // made ordinary, and _, which lodash did. func isWordStart(r rune) bool { return unicode.IsLetter(r) || r == '_' || r == '$' } // isWordRune reports whether a rune can continue one. func isWordRune(r rune) bool { return isWordStart(r) || unicode.IsDigit(r) } // takeWord colours an identifier, deciding what kind of thing it is from the // word itself, from what came before it and from the rune after it. func takeWord(s *syntax.LineScanner, state *lineState) { start := s.Pos() for !s.AtEnd() && isWordRune(s.Peek(0)) { s.Advance(1) } word := wordAt(s, start) class := classOfWord(word, *state, isCallSite(s)) s.Emit(start, s.Pos(), class) if class == syntax.ClassKeyword { state.keyword(word) return } state.operand() } // wordAt returns the text running from start to the scanner's position. func wordAt(s *syntax.LineScanner, start int) string { var b strings.Builder for at := start; at < s.Pos(); at++ { b.WriteRune(s.Peek(at - s.Pos())) } return b.String() } // isCallSite reports whether the next thing on the line, past any spaces, is // an opening parenthesis. func isCallSite(s *syntax.LineScanner) bool { for at := 0; s.Pos()+at < s.Len(); at++ { switch s.Peek(at) { case ' ', '\t': case '(': return true default: return false } } return false } // classOfWord decides what a word is. // // The order is the design. A word after a dot is a property, whatever it is // spelt like — map.get(k) is a call and obj.default a field, and neither is // the keyword it would be on its own. A word after function or class is named // by its position. Then a word the language names is what the language says // it is, except a contextual keyword — get, set, static, of, from, as — that // is being called, which is a function of that name. Then a leading capital // says class, leaning on the convention every JavaScript style guide shares; // then a following parenthesis says function; and what is left is a name. func classOfWord(word string, state lineState, call bool) syntax.Class { if class, decided := classByPosition(state, call); decided { return class } if class, known := knownWords[word]; known { return knownWordClass(word, class, call) } return classBySpelling(word, call) } // classByPosition decides a word by what came before it, and reports whether // that settled it: a property after a dot, a name after function or class. func classByPosition(state lineState, call bool) (syntax.Class, bool) { switch { case state.afterDot: return propertyClass(call), true case state.afterKeyword == "function": return syntax.ClassFunction, true case state.afterKeyword == "class": return syntax.ClassType, true } return syntax.ClassIdentifier, false } // knownWordClass is what a word the language names turns out to be: what the // table says, unless it is a contextual keyword being called — get(k) on a Map // — which is a function of that name. func knownWordClass(word string, class syntax.Class, call bool) syntax.Class { if class == syntax.ClassKeyword && contextual[word] && call { return syntax.ClassFunction } return class } // classBySpelling decides a word nothing else claimed, from its spelling and // the rune after it. // // A leading capital says class — a constructor, a namespace-like object, a // React component. It is visibly a heuristic in one place: a // SCREAMING_SNAKE_CASE constant is coloured as a class too, and the reference // says so rather than leaving somebody to find out. func classBySpelling(word string, call bool) syntax.Class { switch { case startsUpperCase(word): return syntax.ClassType case call: return syntax.ClassFunction } return syntax.ClassIdentifier } // propertyClass is what a word after a dot is: a method when it is called, a // field otherwise. func propertyClass(call bool) syntax.Class { if call { return syntax.ClassFunction } return syntax.ClassIdentifier } // startsUpperCase reports whether a word begins with an ASCII capital. func startsUpperCase(word string) bool { return word != "" && word[0] >= 'A' && word[0] <= 'Z' } // knownWords is every word the language itself names, and what each one is. // // It is one table rather than three because it answers one question. The // groups below are kept apart only so that each can carry the reasoning that // belongs to it. var knownWords = merge( classify(syntax.ClassKeyword, keywords), classify(syntax.ClassConstant, constants), classify(syntax.ClassBuiltin, builtins), ) // contextual are the keywords that are only keywords in some positions, and // ordinary names everywhere else: get and set before a method's name, static // before a member, of in a for-of, from and as in an import. Called — get(k), // set(k, v) — they are functions of that name, on a Map or anywhere else. var contextual = map[string]bool{ "get": true, "set": true, "static": true, "of": true, "from": true, "as": true, } // keywords are JavaScript's reserved words and the contextual ones a reader // meets as keywords — async and await, of, get and set, static, from and as. var keywords = words( "as", "async", "await", "break", "case", "catch", "class", "const", "continue", "debugger", "default", "delete", "do", "else", "export", "extends", "finally", "for", "from", "function", "get", "if", "import", "in", "instanceof", "let", "new", "of", "return", "set", "static", "super", "switch", "throw", "try", "typeof", "var", "void", "while", "with", "yield", // Reserved for future use, and in strict mode. A file using one will not // run, and colouring it as an identifier would be the friendlier of two // wrong answers. "enum", "implements", "interface", "package", "private", "protected", "public", ) // constants are the literals the language itself provides. this is one of // them rather than a keyword: a reader reads it as a value, the one it stands // for, and a slash after it divides. var constants = words("true", "false", "null", "undefined", "NaN", "Infinity", "this") // builtins are the globals worth telling apart from a variable of your own: // the standard library's constructors and namespaces, and — because this is a // Node editor — Node's own globals, from process and Buffer to the CommonJS // five. Recognised by name, as Go's predeclared identifiers are, because a // file may shadow them and colouring the shadowed one anyway is what every // editor does. var builtins = words( // The standard library. "Array", "ArrayBuffer", "BigInt", "Boolean", "DataView", "Date", "Error", "EvalError", "Function", "Intl", "JSON", "Map", "Math", "Number", "Object", "Promise", "Proxy", "RangeError", "ReferenceError", "Reflect", "RegExp", "Set", "String", "Symbol", "SyntaxError", "TypeError", "URIError", "WeakMap", "WeakRef", "WeakSet", "globalThis", "parseInt", "parseFloat", "isNaN", "isFinite", "encodeURIComponent", "decodeURIComponent", "structuredClone", "queueMicrotask", "setTimeout", "clearTimeout", "setInterval", "clearInterval", // Node.js, and the web platform pieces Node ships. "process", "Buffer", "console", "require", "module", "exports", "__dirname", "__filename", "setImmediate", "clearImmediate", "fetch", "URL", "URLSearchParams", "TextEncoder", "TextDecoder", "AbortController", "AbortSignal", "Blob", "Event", "EventTarget", "performance", "crypto", "navigator", // The browser's two, for the front-end half of a Node project. "document", "window", ) // words gathers a group of them, which reads better at the call sites above // than a slice literal does. func words(list ...string) []string { return list } // classify pairs every word in a group with the class it belongs to. func classify(class syntax.Class, list []string) map[string]syntax.Class { out := make(map[string]syntax.Class, len(list)) for _, word := range list { out[word] = class } return out } // merge folds the groups into one table. An earlier group wins a word a later // one repeats, which is what keeps a keyword a keyword. func merge(groups ...map[string]syntax.Class) map[string]syntax.Class { out := map[string]syntax.Class{} for _, group := range groups { for word, class := range group { if _, taken := out[word]; !taken { out[word] = class } } } return out } // Keywords returns the words the scanner colours as keywords, sorted. It is // what a test compares against a real language server's completion. func Keywords() []string { return sorted(keywords) } // Builtins returns the globals the scanner colours as built in, sorted. func Builtins() []string { return sorted(builtins) } // sorted returns a sorted copy of a word list, so a caller cannot reorder the // package's own table. func sorted(list []string) []string { return slices.Sorted(slices.Values(list)) }