1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
|
package jslang
import "rickub.com/turbo-editors/turbo-core/syntax"
// carry is what a line of JavaScript leaves open for the next one.
//
// Two constructs in JavaScript can cross a line break: a block comment, which
// does not nest, and a template literal, which is the one string in the
// language that may hold a real newline. Each is a flag rather than a depth
// for that reason. Everything else — a regular expression, an ordinary string,
// a number — ends on the line it started, by the language's own rules.
type carry struct {
// inComment says a /* ran past the end of a line.
inComment bool
// inTemplate says a `…` template literal ran past the end of a line.
inTemplate bool
// past says at least one line has been scanned, so that a #! on this one
// is two operators rather than the hashbang only the first line may carry.
past bool
}
// lineState is what the scanner knows about the line in front of it that no
// single token says: whether a slash here would begin a regular expression,
// and what the previous significant token was.
//
// It is line-local. A statement that breaks a line between an operator and a
// regular expression is not carried, so a `/` at the start of a line is read
// as a regular expression when one closes on that line, and as division when
// none does.
type lineState struct {
// regexAllowed says a / here opens a regular expression rather than
// dividing: true after an operator, an opening bracket, a keyword and at
// the start of a line; false after anything that can end an expression.
regexAllowed bool
// afterDot says the previous token was a member-access dot, so the word
// here is a property name whatever it is spelt like: obj.default,
// map.get(k), promise.catch(…).
afterDot bool
// afterKeyword is the previous word when it was function or class, so the
// name that follows is coloured by its position.
afterKeyword string
}
// Highlight colours JavaScript source.
//
// It is written against syntax.LineScanner, a line at a time, with the two
// multi-line constructs above threaded through carry. It replaces the scanner
// turbo-core ships for JavaScript: what it adds is regular-expression
// literals, the hashbang line, Node's globals, the name after function and
// class, and a leading capital read as a class name.
//
// It is deliberately tolerant of broken input: source under the cursor is
// invalid most of the time it is being typed, and a highlighter that gives up
// is a highlighter that flickers off.
func Highlight(src string) [][]syntax.Span {
return syntax.ScanLines(src, scanLine)
}
// scanLine colours one line and returns what it leaves open.
func scanLine(line []rune, open carry) ([]syntax.Span, carry) {
s := syntax.NewLineScanner(line)
state := lineState{regexAllowed: true}
// Whatever ran past the end of the previous line is finished first: until
// it closes, nothing on this line is code.
if !finishCarried(s, &open, &state) {
open.past = true
return s.Spans(), open
}
// A hashbang is the first line of a script run as a command. Node strips
// it before parsing, and so does this scanner — on the first line only,
// because anywhere else #! is exactly what it looks like.
if !open.past && s.HasPrefix(0, "#!") {
s.TakeRest(syntax.ClassComment)
}
for !s.AtEnd() {
scanToken(s, &open, &state)
}
open.past = true
return s.Spans(), open
}
// finishCarried closes whatever the previous line left open, and reports
// whether the rest of this line is code.
func finishCarried(s *syntax.LineScanner, open *carry, state *lineState) bool {
switch {
case open.inComment:
if !syntax.FinishBlockComment(s, "*/", syntax.ClassComment) {
return false
}
open.inComment = false
case open.inTemplate:
if !finishTemplate(s) {
return false
}
open.inTemplate = false
// A template that has just closed is a value, so a slash after it
// divides.
state.operand()
}
return true
}
// scanToken colours whatever starts at the scanner's position.
func scanToken(s *syntax.LineScanner, open *carry, state *lineState) {
r := s.Peek(0)
switch {
case r == ' ' || r == '\t':
s.SkipSpaces()
case s.HasPrefix(0, "//"):
s.TakeRest(syntax.ClassComment)
case s.HasPrefix(0, "/*"):
if !syntax.OpenBlockComment(s, "/*", "*/", syntax.ClassComment) {
open.inComment = true
}
case r == '`':
if !openTemplate(s) {
open.inTemplate = true
}
state.operand()
case r == '"' || r == '\'':
syntax.TakeQuoted(s, r, syntax.ClassString)
state.operand()
case r == '/' && state.regexAllowed && takeRegex(s):
// takeRegex consumes nothing when no regular expression closes on
// this line, so the slash falls through to the operator case below.
state.operand()
case syntax.IsDigit(r) || (r == '.' && syntax.IsDigit(s.Peek(1))):
takeNumber(s)
state.operand()
case isWordStart(r):
takeWord(s, state)
case r == '#' && isWordStart(s.Peek(1)):
takePrivateName(s, state)
case r == '@' && isWordStart(s.Peek(1)):
takeDecorator(s, state)
case s.HasPrefix(0, "..."):
// A spread is one thing, not three dots: colouring it as three
// member accesses would also leave afterDot set for the name it
// spreads.
s.Take(3, syntax.ClassPunctuation)
state.punctuation('.')
case s.HasPrefix(0, "?."):
// Optional chaining is a member access with a question in front of
// it, and the name after it is a property like any other.
s.Take(2, syntax.ClassOperator)
state.dot()
case r == '.':
s.Take(1, syntax.ClassPunctuation)
state.dot()
case syntax.IsOperatorRune(r):
start := s.Pos()
s.TakeWhile(syntax.ClassOperator, syntax.IsOperatorRune)
state.operator(wordAt(s, start))
case syntax.IsPunctuationRune(r):
s.Take(1, syntax.ClassPunctuation)
state.punctuation(r)
default:
// A rune nothing here claims — a stray backslash, an emoji outside a
// string — is stepped over uncoloured rather than guessed at.
s.Advance(1)
}
}
// --- what the previous token allows ------------------------------------------
// operand records that the previous token could end an expression, so a slash
// after it divides.
func (st *lineState) operand() {
*st = lineState{}
}
// operator records an operator, after which a slash opens a regular
// expression. The one keyword that survives an operator is function, whose
// generator form puts a * between it and the name: function* gen() {}.
func (st *lineState) operator(text string) {
keyword := ""
if st.afterKeyword == "function" && text == "*" {
keyword = "function"
}
*st = lineState{regexAllowed: true, afterKeyword: keyword}
}
// punctuation records a bracket, a comma or a semicolon. A closing parenthesis
// or bracket ends an expression, so a slash after one divides; anything else
// — including a closing brace, which ends a block far more often than an
// object literal — lets a regular expression begin.
func (st *lineState) punctuation(r rune) {
*st = lineState{regexAllowed: r != ')' && r != ']'}
}
// dot records a member-access dot, after which the next word is a property
// name whatever it is spelt like.
func (st *lineState) dot() {
*st = lineState{afterDot: true}
}
// keyword records a keyword. A slash after return, typeof, case and the rest
// opens a regular expression; function and class are remembered so the name
// after them is coloured by its position.
func (st *lineState) keyword(word string) {
*st = lineState{regexAllowed: true}
if word == "function" || word == "class" {
st.afterKeyword = word
}
}
// --- regular expressions -----------------------------------------------------
// takeRegex colours a regular-expression literal starting at the slash, and
// reports whether there was one.
//
// A literal must close on the line it opened, so the closing slash is looked
// for first — outside a […] class, past any escape — and the slash is left
// alone when none is found: `a /` at the end of a line divides, whatever the
// previous token was. That bound is what makes the previous-token rule safe
// enough to use here where the library declined to: a wrong guess costs at
// most one line, never the rest of the file.
//
// The literal is coloured as a **character literal**. JavaScript has no
// character literal, so the class is free, and a regular expression is the
// other kind of quoted thing in the language — worth telling from a string by
// colour, which is what every theme in the family does with the two classes.
func takeRegex(s *syntax.LineScanner) bool {
at, inClass := 1, false
for s.Pos()+at < s.Len() {
r := s.Peek(at)
switch {
case r == '\\':
at++ // whatever follows a backslash is part of the literal
case r == '[':
inClass = true
case r == ']':
inClass = false
case r == '/' && !inClass:
at++
for isWordRune(s.Peek(at)) { // the flags: /x/gi
at++
}
s.Take(at, syntax.ClassChar)
return true
}
at++
}
return false
}
// --- comments and the words that are not words -------------------------------
// takePrivateName colours a #private class member as the identifier it is,
// the # included: #count is one name and colouring the # alone would make it
// read as a comment marker from another language.
func takePrivateName(s *syntax.LineScanner, state *lineState) {
start := s.Pos()
s.Advance(1)
for !s.AtEnd() && isWordRune(s.Peek(0)) {
s.Advance(1)
}
class := syntax.ClassIdentifier
if isCallSite(s) {
class = syntax.ClassFunction
}
s.Emit(start, s.Pos(), class)
state.operand()
}
// takeDecorator colours a @decorator, dotted path included, as an attribute:
// it is the same kind of thing a Rust #[attribute] is, said about the
// declaration below it, and it is the class the family already uses for that.
func takeDecorator(s *syntax.LineScanner, state *lineState) {
start := s.Pos()
s.Advance(1)
for !s.AtEnd() && (isWordRune(s.Peek(0)) || s.Peek(0) == '.') {
s.Advance(1)
}
s.Emit(start, s.Pos(), syntax.ClassAttribute)
*state = lineState{regexAllowed: true}
}
|