1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
|
package moonbitlang
// Numbers and words: what a run of digits or letters turns out to be.
import (
"strings"
"rickub.com/turbo-editors/turbo-core/syntax"
)
// --- numbers ----------------------------------------------------------------
// numberSuffixes are the literal suffixes MoonBit recognises, longest first so
// that UL is matched before U.
//
// They are upper case and nothing else: the grammar spells out "Uppercase
// suffixes select UInt (U), Int64 (L), UInt64 (UL), BigInt (N), or Float (F)",
// so 123u is the number 123 followed by the identifier u, and colouring it
// otherwise would be inventing a literal the compiler will reject.
var numberSuffixes = []string{"UL", "U", "L", "N", "F"}
// takeNumber colours a numeric literal: 1_000, 0xFF, 0o17, 0b1010, 1.5e-3,
// 0x1.8p3F, 42UL.
//
// It follows the grammar rather than being generous, because one case needs it
// to. "Before .., an integer ends first, so 1..=2 begins with 1 and ..=" — a
// scanner that swallowed any dot would read 1. as a double and leave .=2
// behind, and a range would be miscoloured everywhere it appeared.
//
// The whole literal is one span, so every step here advances the scanner
// without colouring and the single Emit at the end covers what they consumed.
func takeNumber(s *syntax.LineScanner) {
start := s.Pos()
digit, hexadecimal := takeIntegerPart(s)
// A floating-point literal always has a point, and the point is only part
// of the number when a second one does not follow it.
if s.Peek(0) == '.' && s.Peek(1) != '.' {
s.Advance(1)
advanceWhile(s, digit)
}
takeExponent(s, hexadecimal)
takeNumberSuffix(s)
s.Emit(start, s.Pos(), syntax.ClassNumber)
}
// takeIntegerPart consumes the digits before any point. It returns the
// predicate saying which runes count as digits for the rest of the literal, and
// whether the literal is hexadecimal.
//
// The base prefix decides both. After 0x every hexadecimal digit is a digit,
// which is what makes the F of 0x1.F part of the number rather than a Float
// suffix — and an exponent is introduced by p rather than by e, because e is
// itself a hexadecimal digit.
func takeIntegerPart(s *syntax.LineScanner) (digit func(rune) bool, hexadecimal bool) {
if s.Peek(0) == '0' {
switch s.Peek(1) {
case 'x', 'X':
return takeBase(s, isHexDigit), true
case 'o', 'O':
return takeBase(s, isOctalDigit), false
case 'b', 'B':
return takeBase(s, isBinaryDigit), false
}
}
advanceWhile(s, isDecimalDigit)
return isDecimalDigit, false
}
// takeBase consumes a base prefix and the digits after it, and hands back the
// predicate that recognised them.
func takeBase(s *syntax.LineScanner, digit func(rune) bool) func(rune) bool {
s.Advance(2) // the 0 and its base letter
advanceWhile(s, digit)
return digit
}
// takeExponent consumes an exponent when one is there: e or E for a decimal
// literal, p or P for a hexadecimal one, each with an optional sign.
//
// The sign is only taken when a digit follows it, so 1e-x stops at the e and
// leaves the - and the x to be coloured as an operator and a name.
func takeExponent(s *syntax.LineScanner, hexadecimal bool) {
if !isExponentLetter(s.Peek(0), hexadecimal) {
return
}
offset := 1
if s.Peek(offset) == '+' || s.Peek(offset) == '-' {
offset++
}
if !syntax.IsDigit(s.Peek(offset)) {
return
}
s.Advance(offset)
advanceWhile(s, isDecimalDigit)
}
// takeNumberSuffix consumes UL, U, L, N or F when the literal ends in one.
//
// A suffix must not be followed by another word rune: 1Length is not the Int64
// 1 followed by ength, it is a number and then a name the compiler will
// complain about, and stopping short of it is the reading that says so.
func takeNumberSuffix(s *syntax.LineScanner) {
for _, suffix := range numberSuffixes {
if s.HasPrefix(0, suffix) && !syntax.IsWordRune(s.Peek(len(suffix))) {
s.Advance(len(suffix))
return
}
}
}
// advanceWhile steps over runes that match, without colouring any of them. It
// is what a construct emitted as a single span uses in place of TakeWhile,
// which would colour each run it consumed and leave the Emit overlapping it.
func advanceWhile(s *syntax.LineScanner, matches func(rune) bool) {
for !s.AtEnd() && matches(s.Peek(0)) {
s.Advance(1)
}
}
// isExponentLetter reports whether a rune introduces an exponent, which depends
// on the base: a hexadecimal literal uses p, because e is one of its digits.
func isExponentLetter(r rune, hexadecimal bool) bool {
if hexadecimal {
return r == 'p' || r == 'P'
}
return r == 'e' || r == 'E'
}
// isDecimalDigit reports whether a rune may appear in a decimal literal after
// its first digit. An underscore may, and "underscores may repeat or trail".
func isDecimalDigit(r rune) bool { return syntax.IsDigit(r) || r == '_' }
// isHexDigit reports whether a rune may appear in a hexadecimal literal.
func isHexDigit(r rune) bool {
return isDecimalDigit(r) ||
r >= 'a' && r <= 'f' ||
r >= 'A' && r <= 'F'
}
// isOctalDigit reports whether a rune may appear in an octal literal.
func isOctalDigit(r rune) bool { return r >= '0' && r <= '7' || r == '_' }
// isBinaryDigit reports whether a rune may appear in a binary literal.
func isBinaryDigit(r rune) bool { return r == '0' || r == '1' || r == '_' }
// --- words ------------------------------------------------------------------
// bangKeywords are the two keywords that end in an exclamation mark.
//
// The grammar lists try! and guard! among the keywords, and ! is an operator
// rune — so without this the mark would be coloured as an operator hanging off
// the end of a keyword, which is not what the language sees there.
var bangKeywords = map[string]bool{"try": true, "guard": true}
// takeWord colours an identifier, deciding what kind of thing it is from the
// word itself and from the rune that follows it.
func takeWord(s *syntax.LineScanner) {
start := s.Pos()
advanceWhile(s, syntax.IsWordRune)
word := wordAt(s, start)
if bangKeywords[word] && s.Peek(0) == '!' {
s.Advance(1)
s.Emit(start, s.Pos(), syntax.ClassKeyword)
return
}
if isLabel(s, word) {
s.Advance(1) // the ~
s.Emit(start, s.Pos(), syntax.ClassAttribute)
return
}
s.Emit(start, s.Pos(), classOfWord(word, s.Peek(0)))
}
// takeMember colours the name after a dot: a field, or a method.
//
// It is takeWord without the keyword table, because MoonBit's dot-identifiers
// "use the identifier case rules without consulting the keyword table, so .if
// is valid". A record with a field called `type` is ordinary MoonBit, and
// colouring that field as a keyword would be a claim about the language that
// the language contradicts.
func takeMember(s *syntax.LineScanner) {
start := s.Pos()
advanceWhile(s, syntax.IsWordRune)
s.Emit(start, s.Pos(), classOfMember(wordAt(s, start), s.Peek(0)))
}
// wordAt returns the word running from start to the scanner's position.
func wordAt(s *syntax.LineScanner, start int) string {
var b strings.Builder
for at := start; at < s.Pos(); at++ {
b.WriteRune(s.Peek(at - s.Pos()))
}
return b.String()
}
// isLabel reports whether the word just consumed is a labelled argument's name,
// which is to say whether a tilde touches it.
//
// The tilde is MoonBit's alone: it appears in no operator the language has, so
// a tilde against the end of a name can only be a label. The two exclusions are
// the grammar's own — "ASCII-uppercase identifiers and keywords cannot form
// labels" — and they matter, because without the first, Foo~ in a piece of
// half-typed code would colour a type as a label.
func isLabel(s *syntax.LineScanner, word string) bool {
if s.Peek(0) != '~' || word == "" {
return false
}
if startsUpperCase(word) {
return false
}
_, reserved := knownWords[word]
return !reserved
}
// classOfWord decides what a word is, given the rune that follows it.
//
// The order is the design, and MoonBit lets it be shorter than any other
// scanner in this family. A word the language names is what the language says
// it is. After that comes the case rule — and in MoonBit that is a *lexical*
// rule rather than a convention: a uident "begins with an ASCII uppercase
// letter", and only a type, a trait or an enum constructor may be spelt that
// way. So there is no table of built-in types here, and there does not need to
// be: Int, StringBuilder and a type somebody wrote this morning are all
// capitalised, and all coloured by the same line.
//
// What that costs is that a constructor of your own enum is coloured as a type.
// Nothing in the syntax separates Circle(1.0) from a type applied to arguments,
// and inventing a separation would mean being wrong in both directions instead
// of one.
func classOfWord(word string, next rune) syntax.Class {
if class, known := knownWords[word]; known {
return class
}
if startsUpperCase(word) {
return syntax.ClassType
}
if next == '(' {
return syntax.ClassFunction
}
return syntax.ClassIdentifier
}
// classOfMember decides what the name after a dot is. It is classOfWord with
// the keyword table left out; see takeMember.
func classOfMember(word string, next rune) syntax.Class {
if startsUpperCase(word) {
return syntax.ClassType
}
if next == '(' {
return syntax.ClassFunction
}
return syntax.ClassIdentifier
}
// startsUpperCase reports whether a word begins with an ASCII capital, which is
// what makes it a uident.
func startsUpperCase(word string) bool {
return word != "" && word[0] >= 'A' && word[0] <= 'Z'
}
// knownWords is every word the language itself names, and what each one is.
//
// It is one table rather than three because it answers one question. The three
// groups below are kept apart only so that each can carry the reasoning that
// belongs to it.
var knownWords = merge(
classify(syntax.ClassKeyword, keywords),
classify(syntax.ClassConstant, constants),
classify(syntax.ClassBuiltin, builtinValues),
)
// keywords are the words MoonBit reserves, taken from the grammar's keyword
// production.
//
// true and false are in it there and are not here: they are keywords to the
// lexer and values to the reader, and every editor in this family colours them
// as constants. try! and guard! are not here either — the mark is glued on by
// takeWord, because a table cannot hold a word whose last rune is an operator.
//
// package is here although in a .mbt file it is only a *reserved* word, which
// the lexer treats as an identifier and warns about. It is a real keyword in
// the .mbti interface files this editor also colours, and in a .mbt file
// colouring it says exactly what the compiler is about to: this word is not
// yours to use.
//
// The rest of the reserved list — move, ref, static, unsafe, await, and the
// forty others — is deliberately absent. Those are identifiers that earn a
// warning, and a scanner that coloured them as keywords would be telling a
// reader they cannot write `let ref = 1` when they can.
var keywords = words(
"and", "as", "async", "break", "catch", "const", "continue", "declare",
"defer", "derive", "else", "enum", "enumview", "extend", "extenum",
"extern", "fn", "for", "guard", "if", "impl", "import", "in", "is", "let",
"letrec", "lexscan", "loop", "match", "mut", "nobreak", "nocancel",
"noraise", "package", "priv", "proof_assert", "proof_let", "pub", "raise",
"readonly", "return", "struct", "suberror", "test", "throw", "trait",
"try", "type", "using", "where", "while", "with",
)
// constants are the values a reader meets as the language's own.
//
// None, Some, Ok and Err belong to Option and Result rather than to the
// language, but a reader meets them everywhere and reads them as built in, the
// same argument Turbo Rust records for the same four names.
var constants = words("true", "false", "None", "Some", "Ok", "Err")
// builtinValues are the lower-case names the prelude puts in scope without an
// import, read out of moonbitlang/core/prelude rather than remembered.
//
// The prelude's deprecated names — dump, not, tap, then, to_repr — are left
// out on purpose: colouring them as builtins would present as the language's
// own four things it is trying to retire. There is no print here for the same
// kind of reason, and it is the one worth stating: MoonBit has println and has
// never had print, so a table written from habit would have coloured a name
// that does not exist.
var builtinValues = words(
"abort", "assert_eq", "assert_false", "assert_not_eq", "assert_true",
"compare", "debug", "debug_assert", "debug_inspect", "fail", "hash",
"ignore", "inspect", "json_inspect", "null", "panic", "physical_equal",
"println", "repr",
)
// words gathers a group of them, which reads better at the call sites above
// than a slice literal does.
func words(list ...string) []string { return list }
// classify pairs every word in a group with the class it belongs to.
func classify(class syntax.Class, list []string) map[string]syntax.Class {
out := make(map[string]syntax.Class, len(list))
for _, word := range list {
out[word] = class
}
return out
}
// merge folds the groups into one table. An earlier group wins a word a later
// one repeats, which is what keeps a keyword a keyword.
func merge(groups ...map[string]syntax.Class) map[string]syntax.Class {
out := map[string]syntax.Class{}
for _, group := range groups {
for word, class := range group {
if _, taken := out[word]; !taken {
out[word] = class
}
}
}
return out
}
|