| 📦 Turbo JS 91999d1 k33g 12h ago | 1 | package jslang |
| 2 | |
| 3 | // Words: what a run of letters turns out to be, and the tables that decide it. |
| 4 | |
| 5 | import ( |
| 6 | "slices" |
| 7 | "strings" |
| 8 | "unicode" |
| 9 | |
| 10 | "rickub.com/turbo-editors/turbo-core/syntax" |
| 11 | ) |
| 12 | |
| 13 | // isWordStart reports whether a rune can begin an identifier. JavaScript |
| 14 | // identifiers are Unicode: café and 名前 are names, and so is $, which jQuery |
| 15 | // made ordinary, and _, which lodash did. |
| 16 | func isWordStart(r rune) bool { return unicode.IsLetter(r) || r == '_' || r == '$' } |
| 17 | |
| 18 | // isWordRune reports whether a rune can continue one. |
| 19 | func isWordRune(r rune) bool { return isWordStart(r) || unicode.IsDigit(r) } |
| 20 | |
| 21 | // takeWord colours an identifier, deciding what kind of thing it is from the |
| 22 | // word itself, from what came before it and from the rune after it. |
| 23 | func takeWord(s *syntax.LineScanner, state *lineState) { |
| 24 | start := s.Pos() |
| 25 | for !s.AtEnd() && isWordRune(s.Peek(0)) { |
| 26 | s.Advance(1) |
| 27 | } |
| 28 | word := wordAt(s, start) |
| 29 | |
| 30 | class := classOfWord(word, *state, isCallSite(s)) |
| 31 | s.Emit(start, s.Pos(), class) |
| 32 | |
| 33 | if class == syntax.ClassKeyword { |
| 34 | state.keyword(word) |
| 35 | return |
| 36 | } |
| 37 | state.operand() |
| 38 | } |
| 39 | |
| 40 | // wordAt returns the text running from start to the scanner's position. |
| 41 | func wordAt(s *syntax.LineScanner, start int) string { |
| 42 | var b strings.Builder |
| 43 | for at := start; at < s.Pos(); at++ { |
| 44 | b.WriteRune(s.Peek(at - s.Pos())) |
| 45 | } |
| 46 | return b.String() |
| 47 | } |
| 48 | |
| 49 | // isCallSite reports whether the next thing on the line, past any spaces, is |
| 50 | // an opening parenthesis. |
| 51 | func isCallSite(s *syntax.LineScanner) bool { |
| 52 | for at := 0; s.Pos()+at < s.Len(); at++ { |
| 53 | switch s.Peek(at) { |
| 54 | case ' ', '\t': |
| 55 | case '(': |
| 56 | return true |
| 57 | default: |
| 58 | return false |
| 59 | } |
| 60 | } |
| 61 | return false |
| 62 | } |
| 63 | |
| 64 | // classOfWord decides what a word is. |
| 65 | // |
| 66 | // The order is the design. A word after a dot is a property, whatever it is |
| 67 | // spelt like — map.get(k) is a call and obj.default a field, and neither is |
| 68 | // the keyword it would be on its own. A word after function or class is named |
| 69 | // by its position. Then a word the language names is what the language says |
| 70 | // it is, except a contextual keyword — get, set, static, of, from, as — that |
| 71 | // is being called, which is a function of that name. Then a leading capital |
| 72 | // says class, leaning on the convention every JavaScript style guide shares; |
| 73 | // then a following parenthesis says function; and what is left is a name. |
| 74 | func classOfWord(word string, state lineState, call bool) syntax.Class { |
| 75 | if class, decided := classByPosition(state, call); decided { |
| 76 | return class |
| 77 | } |
| 78 | if class, known := knownWords[word]; known { |
| 79 | return knownWordClass(word, class, call) |
| 80 | } |
| 81 | return classBySpelling(word, call) |
| 82 | } |
| 83 | |
| 84 | // classByPosition decides a word by what came before it, and reports whether |
| 85 | // that settled it: a property after a dot, a name after function or class. |
| 86 | func classByPosition(state lineState, call bool) (syntax.Class, bool) { |
| 87 | switch { |
| 88 | case state.afterDot: |
| 89 | return propertyClass(call), true |
| 90 | case state.afterKeyword == "function": |
| 91 | return syntax.ClassFunction, true |
| 92 | case state.afterKeyword == "class": |
| 93 | return syntax.ClassType, true |
| 94 | } |
| 95 | return syntax.ClassIdentifier, false |
| 96 | } |
| 97 | |
| 98 | // knownWordClass is what a word the language names turns out to be: what the |
| 99 | // table says, unless it is a contextual keyword being called — get(k) on a Map |
| 100 | // — which is a function of that name. |
| 101 | func knownWordClass(word string, class syntax.Class, call bool) syntax.Class { |
| 102 | if class == syntax.ClassKeyword && contextual[word] && call { |
| 103 | return syntax.ClassFunction |
| 104 | } |
| 105 | return class |
| 106 | } |
| 107 | |
| 108 | // classBySpelling decides a word nothing else claimed, from its spelling and |
| 109 | // the rune after it. |
| 110 | // |
| 111 | // A leading capital says class — a constructor, a namespace-like object, a |
| 112 | // React component. It is visibly a heuristic in one place: a |
| 113 | // SCREAMING_SNAKE_CASE constant is coloured as a class too, and the reference |
| 114 | // says so rather than leaving somebody to find out. |
| 115 | func classBySpelling(word string, call bool) syntax.Class { |
| 116 | switch { |
| 117 | case startsUpperCase(word): |
| 118 | return syntax.ClassType |
| 119 | case call: |
| 120 | return syntax.ClassFunction |
| 121 | } |
| 122 | return syntax.ClassIdentifier |
| 123 | } |
| 124 | |
| 125 | // propertyClass is what a word after a dot is: a method when it is called, a |
| 126 | // field otherwise. |
| 127 | func propertyClass(call bool) syntax.Class { |
| 128 | if call { |
| 129 | return syntax.ClassFunction |
| 130 | } |
| 131 | return syntax.ClassIdentifier |
| 132 | } |
| 133 | |
| 134 | // startsUpperCase reports whether a word begins with an ASCII capital. |
| 135 | func startsUpperCase(word string) bool { |
| 136 | return word != "" && word[0] >= 'A' && word[0] <= 'Z' |
| 137 | } |
| 138 | |
| 139 | // knownWords is every word the language itself names, and what each one is. |
| 140 | // |
| 141 | // It is one table rather than three because it answers one question. The |
| 142 | // groups below are kept apart only so that each can carry the reasoning that |
| 143 | // belongs to it. |
| 144 | var knownWords = merge( |
| 145 | classify(syntax.ClassKeyword, keywords), |
| 146 | classify(syntax.ClassConstant, constants), |
| 147 | classify(syntax.ClassBuiltin, builtins), |
| 148 | ) |
| 149 | |
| 150 | // contextual are the keywords that are only keywords in some positions, and |
| 151 | // ordinary names everywhere else: get and set before a method's name, static |
| 152 | // before a member, of in a for-of, from and as in an import. Called — get(k), |
| 153 | // set(k, v) — they are functions of that name, on a Map or anywhere else. |
| 154 | var contextual = map[string]bool{ |
| 155 | "get": true, "set": true, "static": true, "of": true, "from": true, "as": true, |
| 156 | } |
| 157 | |
| 158 | // keywords are JavaScript's reserved words and the contextual ones a reader |
| 159 | // meets as keywords — async and await, of, get and set, static, from and as. |
| 160 | var keywords = words( |
| 161 | "as", "async", "await", "break", "case", "catch", "class", "const", |
| 162 | "continue", "debugger", "default", "delete", "do", "else", "export", |
| 163 | "extends", "finally", "for", "from", "function", "get", "if", "import", |
| 164 | "in", "instanceof", "let", "new", "of", "return", "set", "static", "super", |
| 165 | "switch", "throw", "try", "typeof", "var", "void", "while", "with", "yield", |
| 166 | // Reserved for future use, and in strict mode. A file using one will not |
| 167 | // run, and colouring it as an identifier would be the friendlier of two |
| 168 | // wrong answers. |
| 169 | "enum", "implements", "interface", "package", "private", "protected", "public", |
| 170 | ) |
| 171 | |
| 172 | // constants are the literals the language itself provides. this is one of |
| 173 | // them rather than a keyword: a reader reads it as a value, the one it stands |
| 174 | // for, and a slash after it divides. |
| 175 | var constants = words("true", "false", "null", "undefined", "NaN", "Infinity", "this") |
| 176 | |
| 177 | // builtins are the globals worth telling apart from a variable of your own: |
| 178 | // the standard library's constructors and namespaces, and — because this is a |
| 179 | // Node editor — Node's own globals, from process and Buffer to the CommonJS |
| 180 | // five. Recognised by name, as Go's predeclared identifiers are, because a |
| 181 | // file may shadow them and colouring the shadowed one anyway is what every |
| 182 | // editor does. |
| 183 | var builtins = words( |
| 184 | // The standard library. |
| 185 | "Array", "ArrayBuffer", "BigInt", "Boolean", "DataView", "Date", "Error", |
| 186 | "EvalError", "Function", "Intl", "JSON", "Map", "Math", "Number", "Object", |
| 187 | "Promise", "Proxy", "RangeError", "ReferenceError", "Reflect", "RegExp", |
| 188 | "Set", "String", "Symbol", "SyntaxError", "TypeError", "URIError", "WeakMap", |
| 189 | "WeakRef", "WeakSet", "globalThis", |
| 190 | "parseInt", "parseFloat", "isNaN", "isFinite", "encodeURIComponent", |
| 191 | "decodeURIComponent", "structuredClone", "queueMicrotask", |
| 192 | "setTimeout", "clearTimeout", "setInterval", "clearInterval", |
| 193 | // Node.js, and the web platform pieces Node ships. |
| 194 | "process", "Buffer", "console", "require", "module", "exports", |
| 195 | "__dirname", "__filename", "setImmediate", "clearImmediate", |
| 196 | "fetch", "URL", "URLSearchParams", "TextEncoder", "TextDecoder", |
| 197 | "AbortController", "AbortSignal", "Blob", "Event", "EventTarget", |
| 198 | "performance", "crypto", "navigator", |
| 199 | // The browser's two, for the front-end half of a Node project. |
| 200 | "document", "window", |
| 201 | ) |
| 202 | |
| 203 | // words gathers a group of them, which reads better at the call sites above |
| 204 | // than a slice literal does. |
| 205 | func words(list ...string) []string { return list } |
| 206 | |
| 207 | // classify pairs every word in a group with the class it belongs to. |
| 208 | func classify(class syntax.Class, list []string) map[string]syntax.Class { |
| 209 | out := make(map[string]syntax.Class, len(list)) |
| 210 | for _, word := range list { |
| 211 | out[word] = class |
| 212 | } |
| 213 | return out |
| 214 | } |
| 215 | |
| 216 | // merge folds the groups into one table. An earlier group wins a word a later |
| 217 | // one repeats, which is what keeps a keyword a keyword. |
| 218 | func merge(groups ...map[string]syntax.Class) map[string]syntax.Class { |
| 219 | out := map[string]syntax.Class{} |
| 220 | for _, group := range groups { |
| 221 | for word, class := range group { |
| 222 | if _, taken := out[word]; !taken { |
| 223 | out[word] = class |
| 224 | } |
| 225 | } |
| 226 | } |
| 227 | return out |
| 228 | } |
| 229 | |
| 230 | // Keywords returns the words the scanner colours as keywords, sorted. It is |
| 231 | // what a test compares against a real language server's completion. |
| 232 | func Keywords() []string { return sorted(keywords) } |
| 233 | |
| 234 | // Builtins returns the globals the scanner colours as built in, sorted. |
| 235 | func Builtins() []string { return sorted(builtins) } |
| 236 | |
| 237 | // sorted returns a sorted copy of a word list, so a caller cannot reorder the |
| 238 | // package's own table. |
| 239 | func sorted(list []string) []string { return slices.Sorted(slices.Values(list)) } |