1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
|
package rustlang
// Numbers and words: what a run of letters or digits turns out to be.
import (
"strings"
"rickub.com/turbo-editors/turbo-core/syntax"
)
// --- numbers ----------------------------------------------------------------
// takeNumber colours a numeric literal, underscores, base prefix, exponent and
// type suffix included: 1_000, 0xFF_u8, 1.5e-3f64.
//
// The suffix is taken as part of the number rather than as an identifier
// beside it, because 3u8 is one literal and colouring the u8 as a type would
// split a thing that is not two things.
func takeNumber(s *syntax.LineScanner) {
start := s.Pos()
s.Advance(1)
for !s.AtEnd() {
r := s.Peek(0)
switch {
case syntax.IsWordRune(r) || r == '.' && syntax.IsDigit(s.Peek(1)):
s.Advance(1)
case (r == '+' || r == '-') && isExponent(s.Peek(-1)):
s.Advance(1)
default:
s.Emit(start, s.Pos(), syntax.ClassNumber)
return
}
}
s.Emit(start, s.Pos(), syntax.ClassNumber)
}
// isExponent reports whether a rune is the e of an exponent, which is what
// makes the sign after it part of the number rather than an operator.
func isExponent(r rune) bool { return r == 'e' || r == 'E' }
// rangeWidth returns how many runes the range operator at the scanner's
// position takes: three for ..=, two otherwise.
func rangeWidth(s *syntax.LineScanner) int {
if s.Peek(2) == '=' {
return 3
}
return 2
}
// --- words ------------------------------------------------------------------
// takeWord colours an identifier, deciding what kind of thing it is from the
// word itself and from the rune after it.
func takeWord(s *syntax.LineScanner) {
start := s.Pos()
for !s.AtEnd() && syntax.IsWordRune(s.Peek(0)) {
s.Advance(1)
}
word := wordAt(s, start)
end := s.Pos()
// A macro takes the ! with it: println! is one name, and colouring the !
// as an operator would make it read as a negation.
if s.Peek(0) == '!' && s.Peek(1) != '=' {
s.Advance(1)
s.Emit(start, s.Pos(), syntax.ClassBuiltin)
return
}
s.Emit(start, end, classOfWord(word, s.Peek(0)))
}
// wordAt returns the word running from start to the scanner's position.
func wordAt(s *syntax.LineScanner, start int) string {
var b strings.Builder
for at := start; at < s.Pos(); at++ {
b.WriteRune(s.Peek(at - s.Pos()))
}
return b.String()
}
// classOfWord decides what a word is, given the rune that follows it.
//
// The order is the design: a word the language names is what the language says
// it is, whatever follows it — which is what stops `u8::MAX` reading as a call
// — and only then does `(` make a name one.
func classOfWord(word string, next rune) syntax.Class {
if class, known := knownWords[word]; known {
return class
}
if next == '(' {
return syntax.ClassFunction
}
if startsUpperCase(word) {
// Rust's naming convention is strong enough to lean on: a type, a trait
// and an enum variant are all UpperCamelCase and nothing else is, so a
// leading capital says "type" more reliably here than any amount of
// looking at neighbouring tokens would.
return syntax.ClassType
}
return syntax.ClassIdentifier
}
// startsUpperCase reports whether a word begins with an ASCII capital.
func startsUpperCase(word string) bool {
return word != "" && word[0] >= 'A' && word[0] <= 'Z'
}
// knownWords is every word the language itself names, and what each one is.
//
// It is one table rather than three because it answers one question. The three
// groups below are kept apart only so that each can carry the reasoning that
// belongs to it.
var knownWords = merge(
classify(syntax.ClassKeyword, keywords),
classify(syntax.ClassConstant, constants),
classify(syntax.ClassType, primitiveTypes),
)
// keywords are Rust's reserved words, including the ones reserved for future
// use — a file using one will not compile, and colouring it as an identifier
// would be the friendlier of two wrong answers.
var keywords = words(
"as", "async", "await", "break", "const", "continue", "crate", "dyn",
"else", "enum", "extern", "fn", "for", "if", "impl", "in", "let", "loop",
"macro_rules", "match", "mod", "move", "mut", "pub", "ref", "return",
"static", "struct", "super", "trait", "type", "union", "unsafe", "use",
"where", "while", "yield",
// Reserved for future use.
"abstract", "become", "box", "do", "final", "override", "priv", "try",
"typeof", "unsized", "virtual",
)
// constants are the literals the language itself provides, plus the two enum
// variants everybody meets before they meet any other.
//
// None and Some are Option's, not the language's, but a reader looking at Rust
// reads them as they read true and false, and a theme that quiets constants
// should quiet them too.
var constants = words("true", "false", "None", "Some", "Ok", "Err")
// primitiveTypes are the built-in types and the two self words.
//
// self and Self are keywords to the compiler; they are here because what a
// reader wants coloured is the *type* they stand for, and Self in an impl block
// reads as the type it names.
var primitiveTypes = words(
"bool", "char", "str", "f32", "f64",
"i8", "i16", "i32", "i64", "i128", "isize",
"u8", "u16", "u32", "u64", "u128", "usize",
"self", "Self",
)
// words gathers a group of them, which reads better at the call sites above
// than a slice literal does.
func words(list ...string) []string { return list }
// classify pairs every word in a group with the class it belongs to.
func classify(class syntax.Class, list []string) map[string]syntax.Class {
out := make(map[string]syntax.Class, len(list))
for _, word := range list {
out[word] = class
}
return out
}
// merge folds the groups into one table. An earlier group wins a word a later
// one repeats, which is what keeps a keyword a keyword.
func merge(groups ...map[string]syntax.Class) map[string]syntax.Class {
out := map[string]syntax.Class{}
for _, group := range groups {
for word, class := range group {
if _, taken := out[word]; !taken {
out[word] = class
}
}
}
return out
}
|