1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239

package jslang

// Words: what a run of letters turns out to be, and the tables that decide it.

import (
	"slices"
	"strings"
	"unicode"

	"rickub.com/turbo-editors/turbo-core/syntax"
)

// isWordStart reports whether a rune can begin an identifier. JavaScript
// identifiers are Unicode: café and 名前 are names, and so is $, which jQuery
// made ordinary, and _, which lodash did.
func isWordStart(r rune) bool { return unicode.IsLetter(r) || r == '_' || r == '$' }

// isWordRune reports whether a rune can continue one.
func isWordRune(r rune) bool { return isWordStart(r) || unicode.IsDigit(r) }

// takeWord colours an identifier, deciding what kind of thing it is from the
// word itself, from what came before it and from the rune after it.
func takeWord(s *syntax.LineScanner, state *lineState) {
	start := s.Pos()
	for !s.AtEnd() && isWordRune(s.Peek(0)) {
		s.Advance(1)
	}
	word := wordAt(s, start)

	class := classOfWord(word, *state, isCallSite(s))
	s.Emit(start, s.Pos(), class)

	if class == syntax.ClassKeyword {
		state.keyword(word)
		return
	}
	state.operand()
}

// wordAt returns the text running from start to the scanner's position.
func wordAt(s *syntax.LineScanner, start int) string {
	var b strings.Builder
	for at := start; at < s.Pos(); at++ {
		b.WriteRune(s.Peek(at - s.Pos()))
	}
	return b.String()
}

// isCallSite reports whether the next thing on the line, past any spaces, is
// an opening parenthesis.
func isCallSite(s *syntax.LineScanner) bool {
	for at := 0; s.Pos()+at < s.Len(); at++ {
		switch s.Peek(at) {
		case ' ', '\t':
		case '(':
			return true
		default:
			return false
		}
	}
	return false
}

// classOfWord decides what a word is.
//
// The order is the design. A word after a dot is a property, whatever it is
// spelt like — map.get(k) is a call and obj.default a field, and neither is
// the keyword it would be on its own. A word after function or class is named
// by its position. Then a word the language names is what the language says
// it is, except a contextual keyword — get, set, static, of, from, as — that
// is being called, which is a function of that name. Then a leading capital
// says class, leaning on the convention every JavaScript style guide shares;
// then a following parenthesis says function; and what is left is a name.
func classOfWord(word string, state lineState, call bool) syntax.Class {
	if class, decided := classByPosition(state, call); decided {
		return class
	}
	if class, known := knownWords[word]; known {
		return knownWordClass(word, class, call)
	}
	return classBySpelling(word, call)
}

// classByPosition decides a word by what came before it, and reports whether
// that settled it: a property after a dot, a name after function or class.
func classByPosition(state lineState, call bool) (syntax.Class, bool) {
	switch {
	case state.afterDot:
		return propertyClass(call), true
	case state.afterKeyword == "function":
		return syntax.ClassFunction, true
	case state.afterKeyword == "class":
		return syntax.ClassType, true
	}
	return syntax.ClassIdentifier, false
}

// knownWordClass is what a word the language names turns out to be: what the
// table says, unless it is a contextual keyword being called — get(k) on a Map
// — which is a function of that name.
func knownWordClass(word string, class syntax.Class, call bool) syntax.Class {
	if class == syntax.ClassKeyword && contextual[word] && call {
		return syntax.ClassFunction
	}
	return class
}

// classBySpelling decides a word nothing else claimed, from its spelling and
// the rune after it.
//
// A leading capital says class — a constructor, a namespace-like object, a
// React component. It is visibly a heuristic in one place: a
// SCREAMING_SNAKE_CASE constant is coloured as a class too, and the reference
// says so rather than leaving somebody to find out.
func classBySpelling(word string, call bool) syntax.Class {
	switch {
	case startsUpperCase(word):
		return syntax.ClassType
	case call:
		return syntax.ClassFunction
	}
	return syntax.ClassIdentifier
}

// propertyClass is what a word after a dot is: a method when it is called, a
// field otherwise.
func propertyClass(call bool) syntax.Class {
	if call {
		return syntax.ClassFunction
	}
	return syntax.ClassIdentifier
}

// startsUpperCase reports whether a word begins with an ASCII capital.
func startsUpperCase(word string) bool {
	return word != "" && word[0] >= 'A' && word[0] <= 'Z'
}

// knownWords is every word the language itself names, and what each one is.
//
// It is one table rather than three because it answers one question. The
// groups below are kept apart only so that each can carry the reasoning that
// belongs to it.
var knownWords = merge(
	classify(syntax.ClassKeyword, keywords),
	classify(syntax.ClassConstant, constants),
	classify(syntax.ClassBuiltin, builtins),
)

// contextual are the keywords that are only keywords in some positions, and
// ordinary names everywhere else: get and set before a method's name, static
// before a member, of in a for-of, from and as in an import. Called — get(k),
// set(k, v) — they are functions of that name, on a Map or anywhere else.
var contextual = map[string]bool{
	"get": true, "set": true, "static": true, "of": true, "from": true, "as": true,
}

// keywords are JavaScript's reserved words and the contextual ones a reader
// meets as keywords — async and await, of, get and set, static, from and as.
var keywords = words(
	"as", "async", "await", "break", "case", "catch", "class", "const",
	"continue", "debugger", "default", "delete", "do", "else", "export",
	"extends", "finally", "for", "from", "function", "get", "if", "import",
	"in", "instanceof", "let", "new", "of", "return", "set", "static", "super",
	"switch", "throw", "try", "typeof", "var", "void", "while", "with", "yield",
	// Reserved for future use, and in strict mode. A file using one will not
	// run, and colouring it as an identifier would be the friendlier of two
	// wrong answers.
	"enum", "implements", "interface", "package", "private", "protected", "public",
)

// constants are the literals the language itself provides. this is one of
// them rather than a keyword: a reader reads it as a value, the one it stands
// for, and a slash after it divides.
var constants = words("true", "false", "null", "undefined", "NaN", "Infinity", "this")

// builtins are the globals worth telling apart from a variable of your own:
// the standard library's constructors and namespaces, and — because this is a
// Node editor — Node's own globals, from process and Buffer to the CommonJS
// five. Recognised by name, as Go's predeclared identifiers are, because a
// file may shadow them and colouring the shadowed one anyway is what every
// editor does.
var builtins = words(
	// The standard library.
	"Array", "ArrayBuffer", "BigInt", "Boolean", "DataView", "Date", "Error",
	"EvalError", "Function", "Intl", "JSON", "Map", "Math", "Number", "Object",
	"Promise", "Proxy", "RangeError", "ReferenceError", "Reflect", "RegExp",
	"Set", "String", "Symbol", "SyntaxError", "TypeError", "URIError", "WeakMap",
	"WeakRef", "WeakSet", "globalThis",
	"parseInt", "parseFloat", "isNaN", "isFinite", "encodeURIComponent",
	"decodeURIComponent", "structuredClone", "queueMicrotask",
	"setTimeout", "clearTimeout", "setInterval", "clearInterval",
	// Node.js, and the web platform pieces Node ships.
	"process", "Buffer", "console", "require", "module", "exports",
	"__dirname", "__filename", "setImmediate", "clearImmediate",
	"fetch", "URL", "URLSearchParams", "TextEncoder", "TextDecoder",
	"AbortController", "AbortSignal", "Blob", "Event", "EventTarget",
	"performance", "crypto", "navigator",
	// The browser's two, for the front-end half of a Node project.
	"document", "window",
)

// words gathers a group of them, which reads better at the call sites above
// than a slice literal does.
func words(list ...string) []string { return list }

// classify pairs every word in a group with the class it belongs to.
func classify(class syntax.Class, list []string) map[string]syntax.Class {
	out := make(map[string]syntax.Class, len(list))
	for _, word := range list {
		out[word] = class
	}
	return out
}

// merge folds the groups into one table. An earlier group wins a word a later
// one repeats, which is what keeps a keyword a keyword.
func merge(groups ...map[string]syntax.Class) map[string]syntax.Class {
	out := map[string]syntax.Class{}
	for _, group := range groups {
		for word, class := range group {
			if _, taken := out[word]; !taken {
				out[word] = class
			}
		}
	}
	return out
}

// Keywords returns the words the scanner colours as keywords, sorted. It is
// what a test compares against a real language server's completion.
func Keywords() []string { return sorted(keywords) }

// Builtins returns the globals the scanner colours as built in, sorted.
func Builtins() []string { return sorted(builtins) }

// sorted returns a sorted copy of a word list, so a caller cannot reorder the
// package's own table.
func sorted(list []string) []string { return slices.Sorted(slices.Values(list)) }