1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263

package pythonlang

// Numbers and words: what a run of letters or digits turns out to be.

import (
	"strings"

	"rickub.com/turbo-editors/turbo-core/syntax"
)

// --- numbers ----------------------------------------------------------------

// takeNumber colours a numeric literal, base prefix, underscores, exponent and
// imaginary suffix included: 1_000, 0xFF, 0b1010, .5, 1.5e-3, 3j.
func takeNumber(s *syntax.LineScanner) {
	start := s.Pos()
	// A float may have exactly one dot, and 1. is as valid as 1.0 — so the dot
	// is counted rather than required to have a digit after it. A second one is
	// where the number stops, which is what keeps `1..2` from being one token.
	seenDot := s.Peek(0) == '.'
	// 0xE-1 is a hexadecimal literal minus one, not an exponent: the sign rule
	// below has to know that the E it just saw was a digit.
	hex := s.Peek(0) == '0' && (s.Peek(1) == 'x' || s.Peek(1) == 'X')
	s.Advance(1)

	for !s.AtEnd() {
		r := s.Peek(0)
		switch {
		case syntax.IsWordRune(r):
			s.Advance(1)
		case r == '.' && !seenDot:
			seenDot = true
			s.Advance(1)
		case (r == '+' || r == '-') && !hex && isExponent(s.Peek(-1)):
			s.Advance(1)
		default:
			s.Emit(start, s.Pos(), syntax.ClassNumber)
			return
		}
	}
	s.Emit(start, s.Pos(), syntax.ClassNumber)
}

// isExponent reports whether a rune is the e of an exponent, which is what
// makes the sign after it part of the number rather than an operator.
func isExponent(r rune) bool { return r == 'e' || r == 'E' }

// --- words ------------------------------------------------------------------

// takeWord colours an identifier, deciding what kind of thing it is from the
// word itself, from the rune after it, and — for the two soft keywords — from
// where it sits on the line.
func takeWord(s *syntax.LineScanner) {
	start := s.Pos()
	// Asked before the word is consumed, because afterwards the scanner is no
	// longer at its first rune.
	first := atLineStart(s)

	for !s.AtEnd() && syntax.IsWordRune(s.Peek(0)) {
		s.Advance(1)
	}
	word := wordAt(s, start)

	class := classOfWord(word, s.Peek(0))
	if isSoftKeyword(word) && first && lineEndsWithColon(s) {
		class = syntax.ClassKeyword
	}
	s.Emit(start, s.Pos(), class)
}

// wordAt returns the word running from start to the scanner's position.
func wordAt(s *syntax.LineScanner, start int) string {
	var b strings.Builder
	for at := start; at < s.Pos(); at++ {
		b.WriteRune(s.Peek(at - s.Pos()))
	}
	return b.String()
}

// isSoftKeyword reports whether a word is one of the two Python added without
// reserving.
//
// `match` and `case` open a match statement, and are ordinary names everywhere
// else — `match = re.match(pattern, text)` is the line that made this a rule
// rather than a table entry. What tells them apart is the shape of a
// statement: it starts the line, and the line ends with the colon that opens
// its block. Both conditions are checked, and `type` is deliberately not here:
// it is a builtin as well as a soft keyword, and reading as a builtin is right
// in both of its jobs.
func isSoftKeyword(word string) bool { return word == "match" || word == "case" }

// classOfWord decides what a word is, given the rune that follows it.
//
// The order is the design. A word the language names is what the language says
// it is, whatever follows it. Then the naming conventions, which in Python are
// strong enough to answer a question that the syntax cannot: a class is called
// exactly the way a function is, so `ValueError("nope")` and `parse("nope")`
// are the same shape, and only CapWords tells them apart. Only after that does
// a parenthesis make a name a function.
//
// This is the one place Turbo Python and Turbo Rust order the same three rules
// differently, and the reason is Rust's `Some(x)`: there, a parenthesis after a
// capitalised name is usually a constructor the language names, so it is
// answered from the table before the convention is consulted.
func classOfWord(word string, next rune) syntax.Class {
	if class, known := knownWords[word]; known {
		return class
	}
	for _, convention := range conventions {
		if convention.spelt(word) {
			return convention.class
		}
	}
	if next == '(' {
		return syntax.ClassFunction
	}
	return syntax.ClassIdentifier
}

// conventions are the ways Python spells what a name is, consulted in order for
// a word the language does not name itself.
//
// A list rather than a chain of ifs because the order *is* the rule, and a list
// is where a reader looks for one: a dunder is the language's whatever else it
// looks like, and a name in capitals is a constant before it is a type.
var conventions = []struct {
	spelt func(string) bool
	class syntax.Class
}{
	{isDunder, syntax.ClassBuiltin},
	{isScreamingCase, syntax.ClassConstant},
	{startsUpperCase, syntax.ClassType},
}

// isDunder reports whether a word is one of the names the language reserves to
// itself by spelling: __init__, __name__, __repr__.
//
// They are the language's own hooks rather than anybody's identifiers, and a
// reader looking for where a class begins finds __init__ faster when it is not
// the same colour as the method below it.
func isDunder(word string) bool {
	return len(word) > 4 && strings.HasPrefix(word, "__") && strings.HasSuffix(word, "__")
}

// isScreamingCase reports whether a word is written the way PEP 8 writes a
// constant: MAX_SIZE, HTTP_PORT, PI.
//
// Turbo Rust has no rule like this one and documents SCREAMING_SNAKE_CASE as a
// known wrong answer — it colours such a word as a type. Python's convention is
// separated from its class convention by more than Rust's is, so the wrong
// answer is worth removing rather than inheriting. What it costs is a class
// named in capitals, which is rare enough to document.
func isScreamingCase(word string) bool {
	if len(word) < 2 {
		return false
	}
	letters := 0
	for _, r := range word {
		switch {
		case r >= 'A' && r <= 'Z':
			letters++
		case r == '_' || r >= '0' && r <= '9':
		default:
			return false
		}
	}
	return letters > 0
}

// startsUpperCase reports whether a word begins with an ASCII capital.
func startsUpperCase(word string) bool {
	return word != "" && word[0] >= 'A' && word[0] <= 'Z'
}

// knownWords is every word the language itself names, and what each one is.
//
// It is one table rather than four because it answers one question. The four
// groups below are kept apart only so that each can carry the reasoning that
// belongs to it.
//
// The exception hierarchy is deliberately absent. ValueError, KeyError and the
// seventy others are CapWords, so the convention rule in classOfWord already
// colours them as types — and a table naming them would go out of date the next
// time Python adds one.
var knownWords = merge(
	classify(syntax.ClassKeyword, keywords),
	classify(syntax.ClassConstant, constants),
	classify(syntax.ClassType, builtinTypes),
	classify(syntax.ClassBuiltin, builtinFunctions),
)

// keywords are Python's reserved words — the ones that cannot be used as a
// name. and, or, not, in and is are here rather than among the operators
// because that is what the language calls them, and because a theme that quiets
// keywords should quiet them.
//
// match and case are not here: they are reserved in no context at all, and are
// decided by isSoftKeyword.
var keywords = words(
	"and", "as", "assert", "async", "await", "break", "class", "continue",
	"def", "del", "elif", "else", "except", "finally", "for", "from", "global",
	"if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass",
	"raise", "return", "try", "while", "with", "yield",
)

// constants are the values the language names, plus the flag it sets for you.
var constants = words("True", "False", "None", "NotImplemented", "Ellipsis", "__debug__")

// builtinTypes are the types you can call without importing anything.
//
// self and cls are here as *builtins* rather than as types, in builtinFunctions
// below — they name a value, not a type.
var builtinTypes = words(
	"bool", "bytearray", "bytes", "complex", "dict", "float", "frozenset",
	"int", "list", "memoryview", "object", "range", "set", "slice", "str",
	"tuple", "type",
)

// builtinFunctions are the names in the builtins module, plus the two argument
// names every Python reader reads as the language's own.
//
// self and cls are a convention rather than a rule — a method may name its
// first parameter anything — but the convention is universal enough that every
// other highlighter colours them, and a reader who meets `self` reads it the
// way a Rust reader reads `Some`. That parallel is the argument; the caveat is
// that a parameter honestly named self in a plain function is coloured too.
var builtinFunctions = words(
	"abs", "aiter", "anext", "all", "any", "ascii", "bin", "breakpoint",
	"callable", "chr", "classmethod", "compile", "delattr", "dir", "divmod",
	"enumerate", "eval", "exec", "filter", "format", "getattr", "globals",
	"hasattr", "hash", "help", "hex", "id", "input", "isinstance", "issubclass",
	"iter", "len", "locals", "map", "max", "min", "next", "oct", "open", "ord",
	"pow", "print", "property", "repr", "reversed", "round", "setattr",
	"sorted", "staticmethod", "sum", "super", "vars", "zip",
	"self", "cls",
)

// words gathers a group of them, which reads better at the call sites above
// than a slice literal does.
func words(list ...string) []string { return list }

// classify pairs every word in a group with the class it belongs to.
func classify(class syntax.Class, list []string) map[string]syntax.Class {
	out := make(map[string]syntax.Class, len(list))
	for _, word := range list {
		out[word] = class
	}
	return out
}

// merge folds the groups into one table. An earlier group wins a word a later
// one repeats, which is what keeps a keyword a keyword.
func merge(groups ...map[string]syntax.Class) map[string]syntax.Class {
	out := map[string]syntax.Class{}
	for _, group := range groups {
		for word, class := range group {
			if _, taken := out[word]; !taken {
				out[word] = class
			}
		}
	}
	return out
}