1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
|
package gololang
// Numbers and words: what a run of digits or letters turns out to be.
import (
"strings"
"unicode"
"rickub.com/turbo-editors/turbo-core/syntax"
)
// --- numbers ----------------------------------------------------------------
// takeNumber colours a numeric literal the way lexer.go's readNumber reads
// one: digits, then a point and more digits if a digit follows the point, then
// an exponent with an optional sign, then the L that makes a long or the F or
// f that makes a float.
//
// The point is only part of the number when a digit follows it. That is what
// keeps 1..3 a number and a range rather than the double 1. and a stray .3,
// and it is the lexer's own test — `l.ch == '.' && isDigit(l.peekChar())`.
//
// There is no hexadecimal, no binary, no octal and no digit separator, because
// the lexer has none: 0xFF is the number 0 followed by the name xFF, and 1_000
// is 1 followed by the name _000. Colouring either as one number would be
// inventing a literal the interpreter will reject.
//
// The whole literal is one span, so every step here advances the scanner
// without colouring and the single Emit at the end covers what they consumed.
func takeNumber(s *syntax.LineScanner) {
start := s.Pos()
advanceWhile(s, syntax.IsDigit)
if s.Peek(0) == '.' && syntax.IsDigit(s.Peek(1)) {
s.Advance(1)
advanceWhile(s, syntax.IsDigit)
}
takeExponent(s)
takeNumberSuffix(s)
s.Emit(start, s.Pos(), syntax.ClassNumber)
}
// takeExponent consumes e or E, an optional sign, and the digits after them.
//
// The digits may be none. The lexer reads 1e as a float and leaves the parser
// to complain, and this scanner colours what the lexer reads.
func takeExponent(s *syntax.LineScanner) {
if s.Peek(0) != 'e' && s.Peek(0) != 'E' {
return
}
s.Advance(1)
if s.Peek(0) == '+' || s.Peek(0) == '-' {
s.Advance(1)
}
advanceWhile(s, syntax.IsDigit)
}
// takeNumberSuffix consumes the L of a long and the F or f of a float, in
// that order, when the literal ends in them. 42L is a long; 3.14F and 2.0f are
// floats; the lexer accepts both cases of the F and only the upper case of
// the L.
func takeNumberSuffix(s *syntax.LineScanner) {
if s.Peek(0) == 'L' {
s.Advance(1)
}
if s.Peek(0) == 'F' || s.Peek(0) == 'f' {
s.Advance(1)
}
}
// advanceWhile steps over runes that match, without colouring any of them. It
// is what a construct emitted as a single span uses in place of TakeWhile,
// which would colour each run it consumed and leave the Emit overlapping it.
func advanceWhile(s *syntax.LineScanner, matches func(rune) bool) {
for !s.AtEnd() && matches(s.Peek(0)) {
s.Advance(1)
}
}
// --- words ------------------------------------------------------------------
// isIdentifierStart reports whether a rune may begin a name, by the lexer's
// own isLetter: any Unicode letter or mark, an underscore, or an emoji.
//
// This is wider than turbo-core's ASCII IsLetter on purpose. Golo lets you
// write `let 😀 = 1` and `function 🚀launch = { … }`, and a scanner that left
// those uncoloured would be telling a reader they are not names when the
// interpreter says they are.
func isIdentifierStart(r rune) bool {
return unicode.IsLetter(r) || unicode.IsMark(r) || r == '_' || isEmoji(r)
}
// isWordRune reports whether a rune may continue a name: whatever may begin
// one, or a digit.
func isWordRune(r rune) bool {
return isIdentifierStart(r) || syntax.IsDigit(r)
}
// emojiBlocks are the four Unicode blocks the lexer admits into a name, each
// as its first and last rune. They are the lexer's own, copied rather than
// widened: the general symbol blocks are left out there because they hold the
// operators.
var emojiBlocks = [][2]rune{
{0x1F600, 0x1F64F}, // emoticons
{0x1F300, 0x1F5FF}, // miscellaneous symbols and pictographs
{0x1F680, 0x1F6FF}, // transport and map symbols
{0x1F900, 0x1F9FF}, // supplemental symbols and pictographs
}
// isEmoji reports whether a rune is in one of the blocks the lexer admits into
// a name.
func isEmoji(r rune) bool {
for _, block := range emojiBlocks {
if r >= block[0] && r <= block[1] {
return true
}
}
return false
}
// takeWord colours a name, deciding what kind of thing it is from the word
// itself and from the rune that follows it — and, for two keywords, colours
// the name that follows *them*, because that name means something only there.
func takeWord(s *syntax.LineScanner) {
start := s.Pos()
advanceWhile(s, isWordRune)
word := wordAt(s, start)
s.Emit(start, s.Pos(), classOfWord(word, s.Peek(0)))
switch word {
case "module", "import":
takeModulePath(s)
case "function":
takeDeclaredFunctionName(s)
}
}
// takeModulePath colours the dotted name after module or import as one span:
// hello.World, gololang.Errors, java.util.List.
//
// It is one span because it is one name — a module has no parts a program can
// take apart — and ClassType is the nearest of the seventeen classes: a module
// path names a thing rather than holding a value, and the reading it has to be
// saved from is the one where gololang.Errors looks like a variable called
// gololang with something done to it.
//
// A dot is only taken when a name follows it, so `import a.` at the end of a
// half-typed line stops before the dot and leaves it as punctuation.
func takeModulePath(s *syntax.LineScanner) {
s.SkipSpaces()
if !isIdentifierStart(s.Peek(0)) {
return
}
start := s.Pos()
advanceWhile(s, isWordRune)
for s.Peek(0) == '.' && isIdentifierStart(s.Peek(1)) {
s.Advance(1)
advanceWhile(s, isWordRune)
}
s.Emit(start, s.Pos(), syntax.ClassType)
}
// takeDeclaredFunctionName colours the name after the function keyword as a
// function.
//
// Everywhere else a name is a function because a parenthesis follows it, and
// a declaration is the one place that is not true: `function main = |args|`
// has the name followed by an equals sign. Without this, every function a
// file declares would be coloured as an ordinary variable at the one place a
// reader looks for it.
func takeDeclaredFunctionName(s *syntax.LineScanner) {
s.SkipSpaces()
if !isIdentifierStart(s.Peek(0)) {
return
}
start := s.Pos()
advanceWhile(s, isWordRune)
s.Emit(start, s.Pos(), syntax.ClassFunction)
}
// wordAt returns the word running from start to the scanner's position.
func wordAt(s *syntax.LineScanner, start int) string {
var b strings.Builder
for at := start; at < s.Pos(); at++ {
b.WriteRune(s.Peek(at - s.Pos()))
}
return b.String()
}
// classOfWord decides what a word is, given the rune that follows it.
//
// The order is the design. A word the language names is what the language
// says it is: a keyword, one of the three literal constants, or one of the
// interpreter's built-in functions. After that comes the case rule, which in
// Golo is a convention rather than a lexical fact: structs, unions and their
// variants are capitalised by everybody — Point, Shape, Circle, Some, None —
// and nothing else customarily is, so a capitalised word is coloured as a
// type. A lower-case word followed by a parenthesis is a call, and anything
// else is a name.
//
// What the case rule costs is that a variant's constructor is coloured as a
// type — Circle(1.0) and Result_Failure("no") look like types applied to
// arguments — and a capitalised variable, which Golo permits, is coloured as
// one too. Nothing in the syntax separates them, and inventing a separation
// would mean being wrong in both directions instead of one.
func classOfWord(word string, next rune) syntax.Class {
if class, known := knownWords[word]; known {
return class
}
if startsUpperCase(word) {
return syntax.ClassType
}
if next == '(' {
return syntax.ClassFunction
}
return syntax.ClassIdentifier
}
// startsUpperCase reports whether a word begins with an ASCII capital, which
// is what the convention means by a type's name.
func startsUpperCase(word string) bool {
return word != "" && word[0] >= 'A' && word[0] <= 'Z'
}
// knownWords is every word the language itself names, and what each one is.
//
// It is one table rather than three because it answers one question. The three
// groups below are kept apart only so that each can carry the reasoning that
// belongs to it.
var knownWords = merge(
classify(syntax.ClassKeyword, keywords),
classify(syntax.ClassConstant, constants),
classify(syntax.ClassBuiltin, builtinFunctions),
)
// keywords are the words Golo reserves, taken from token/token.go's keyword
// table in GoloScript — every entry of it except the three literal values,
// which are constants below.
//
// The word operators are here as keywords: and, or, not, is, isnt, oftype and
// orIfNull are reserved words that happen to compute something, and a reader
// meets them as words. There is no `then`-less if or `elseif`; `else if` is
// two keywords.
var keywords = words(
"and", "augment", "augmentation", "await", "break", "case", "catch",
"continue", "else", "finally", "for", "foreach", "function", "if",
"import", "in", "is", "isnt", "let", "local", "match", "module", "not",
"oftype", "or", "orIfNull", "otherwise", "return", "spawn", "struct",
"then", "throw", "try", "union", "var", "when", "while", "with",
)
// constants are the values a reader meets as the language's own. They are
// keywords to the lexer and values to the reader, and every editor in this
// family colours them as constants.
var constants = words("true", "false", "null")
// builtinFunctions are the functions the interpreter provides without an
// import, read out of evaluator.BuiltinNames() in GoloScript rather than
// remembered — 162 names, less the five prefixed with a double underscore,
// which are the test runner's own counters and which the language server
// likewise keeps out of its completion list.
//
// Some, None, Ok and Err are deliberately not here. In Golo they are not built
// in: they are the variants of ordinary unions declared in gololang.Errors,
// available only after `import gololang.Errors`, so they take the colour every
// other capitalised name does. DynamicObject *is* here, capital and all,
// because it is a builtin function that happens to be spelt like a type.
var builtinFunctions = words(
"DynamicObject", "abs", "appendFile", "array", "chanClose", "chanReceive",
"chanSend", "channel", "currentAbsDir", "currentDir", "currentTime",
"currentTimeMillis", "currentTimeNano", "dateString", "dateTimeString",
"deleteFile", "escapeJSON", "execCombinedOutput", "execCommand",
"fileExists", "fileInfo", "float", "formatTime", "fromJSON", "getenv",
"head", "httpDelete", "httpDeleteStream", "httpGet", "httpGetStream",
"httpPost", "httpPostStream", "httpPut", "httpPutStream", "httpServe",
"httpStop", "information", "int", "isNotNull", "isNull", "len", "length",
"list", "listDir", "map", "mcpAddResource", "mcpAddTool", "mcpCallTool",
"mcpConnectHTTP", "mcpConnectStdio", "mcpCreateServer", "mcpDisconnect",
"mcpListResources", "mcpListTools", "mcpReadResource", "mcpRunHTTP",
"mcpRunStdio", "mcpStopServer", "mkDir", "mutex", "mutexLock",
"mutexUnlock", "now", "observable", "observableFilter", "observableGet",
"observableMap", "observableOnChange", "observableSet",
"openAIChatCompletion", "openAIChatCompletionStream",
"openAICreateEmbedding", "openAINewClient", "parseTime", "pow", "print",
"println", "push", "raise", "range", "read", "readFile", "readln",
"require", "requireNotNull", "set", "setenv", "sharedGet", "sharedSet",
"sharedState", "sharedUpdate", "sleep", "sqrt", "str", "tail", "template",
"timeString", "toJSON", "tuiClick", "tuiComponentIds", "tuiDisplayWidth",
"tuiEmit", "tuiFocus", "tuiFrame", "tuiGetProp", "tuiHasComponent",
"tuiHide", "tuiIsFocused", "tuiIsVisible", "tuiLoad", "tuiLoadStyle",
"tuiNew", "tuiOff", "tuiOn", "tuiQuit", "tuiRenderMarkdown", "tuiRun",
"tuiSetProp", "tuiShow", "tuiSize", "tuiWheel", "tuiZoneOf",
"tupleFromArray", "type", "uiConfirm", "uiError", "uiGetColorCode",
"uiInfo", "uiMarkdownRender", "uiMarkdownStreamAppend",
"uiMarkdownStreamEnd", "uiMarkdownStreamStart", "uiPrint",
"uiPrintMultiStyle", "uiPrintln", "uiPrompt", "uiPromptMultiline",
"uiPromptPassword", "uiSpinnerError", "uiSpinnerNew", "uiSpinnerSetFrames",
"uiSpinnerSetPrefix", "uiSpinnerSetSuffix", "uiSpinnerStart",
"uiSpinnerStop", "uiSpinnerSuccess", "uiSuccess", "uiWarning", "vector",
"wasmCallNumbers", "wasmCallString", "wasmClose", "wasmHasFunction",
"wasmLoad", "wasmRegisterStringHandler", "wasmShutdown", "writeFile",
)
// Builtins returns the interpreter's built-in function names, sorted, so a
// test can hold the table above to what a real golo answers.
func Builtins() []string {
out := make([]string, len(builtinFunctions))
copy(out, builtinFunctions)
return out
}
// Keywords returns the reserved words the scanner colours as keywords, so a
// test can hold the table above to what a real golo reserves.
func Keywords() []string {
out := make([]string, len(keywords))
copy(out, keywords)
return out
}
// words gathers a group of them, which reads better at the call sites above
// than a slice literal does.
func words(list ...string) []string { return list }
// classify pairs every word in a group with the class it belongs to.
func classify(class syntax.Class, list []string) map[string]syntax.Class {
out := make(map[string]syntax.Class, len(list))
for _, word := range list {
out[word] = class
}
return out
}
// merge folds the groups into one table. An earlier group wins a word a later
// one repeats, which is what keeps a keyword a keyword.
func merge(groups ...map[string]syntax.Class) map[string]syntax.Class {
out := map[string]syntax.Class{}
for _, group := range groups {
for word, class := range group {
if _, taken := out[word]; !taken {
out[word] = class
}
}
}
return out
}
|