1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
package pythonlang
import "rickub.com/turbo-editors/turbo-core/syntax"
// carry is what a line of Python leaves open for the next one.
//
// Only one construct in Python crosses a line break, and it does so in two
// ways. A triple-quoted string runs until the matching three quotes, however
// many lines away that is. A single-quoted one runs on only when the line ends
// with a backslash, which escapes the newline — anything else that reaches the
// end of a line with a quote still open is broken source, and is coloured to
// the end of that line and dropped rather than painting the rest of the file.
//
// Which quote opened it has to be remembered rather than guessed: a literal
// opened with three double quotes and one opened with three apostrophes are
// different strings, and the closer of one appearing inside the other closes
// nothing. (Spelling those triples out in words is deliberate — gofmt rewrites
// a bare run of apostrophes in a doc comment into a typographic quote.)
//
// Rawness is deliberately *not* carried. `r"\""` is a complete string: in a raw
// string the backslash stays in the value, but it still stops the quote after
// it from ending the literal. Termination is therefore the same rule for both,
// and a flag saying otherwise would be a flag nothing reads.
type carry struct {
// open says a string ran past the end of a line.
open bool
// quote is the rune that opened it, ' or ".
quote rune
// triple says it was opened with three of them.
triple bool
}
// Highlight colours Python source.
//
// It is written against syntax.LineScanner, a line at a time, with the one
// multi-line construct above threaded through carry. Python has no tokeniser in
// the Go standard library the way Go does, so this is a scanner in the same
// style as the ones turbo-core ships for TOML, Markdown and shell.
//
// It is deliberately tolerant of broken input: source under the cursor is
// invalid most of the time it is being typed, and a highlighter that gives up
// is a highlighter that flickers off.
func Highlight(src string) [][]syntax.Span {
return syntax.ScanLines(src, scanLine)
}
// scanLine colours one line and returns what it leaves open.
func scanLine(line []rune, open carry) ([]syntax.Span, carry) {
s := syntax.NewLineScanner(line)
// Whatever ran past the end of the previous line is finished first: until
// it closes, nothing on this line is code.
if open.open && !continueString(s, &open) {
return s.Spans(), open
}
for !s.AtEnd() {
scanToken(s, &open)
}
return s.Spans(), open
}
// scanToken colours whatever starts at the scanner's position.
func scanToken(s *syntax.LineScanner, open *carry) {
r := s.Peek(0)
switch {
case r == ' ' || r == '\t':
s.SkipSpaces()
case r == '#':
// Python has no block comment. A run of # lines is a run of comments,
// and a """docstring""" is a string, which is what the language calls
// it and what `help()` reads back.
s.TakeRest(syntax.ClassComment)
case isStringStart(s):
// Before words, because f, r, b and u are letters: without this,
// f"{name}" would be an identifier followed by a string.
takeString(s, open)
case isDecoratorStart(s):
takeDecorator(s)
case syntax.IsDigit(r) || r == '.' && syntax.IsDigit(s.Peek(1)):
// .5 is a float, so a dot with a digit after it starts a number. A dot
// with anything else after it is an attribute access.
takeNumber(s)
case syntax.IsLetter(r) || r == '_':
takeWord(s)
case r == ':' && s.Peek(1) != '=':
// A colon opens a block, separates a dict's key from its value, cuts a
// slice and introduces an annotation: structure in every case, so it
// goes with the brackets and the commas rather than with the
// arithmetic. ":" is an operator rune, so this has to come first — and
// it has to let ":=" through, which really is an operator.
s.Take(1, syntax.ClassPunctuation)
case r == '@':
// Not a decorator, or the case above would have taken it: this is the
// matrix-multiplication operator.
s.Take(1, syntax.ClassOperator)
case r == '\\':
// A backslash at the end of a line joins it to the next one. It is
// structure rather than computation, and colouring it says that the
// line does not end where it looks like it ends.
s.Take(1, syntax.ClassPunctuation)
case syntax.IsOperatorRune(r):
s.TakeWhile(syntax.ClassOperator, syntax.IsOperatorRune)
case syntax.IsPunctuationRune(r):
s.Take(1, syntax.ClassPunctuation)
default:
// A rune nothing here claims — a currency sign in a comment-free line,
// an accented letter in an identifier — is stepped over uncoloured
// rather than guessed at.
s.Advance(1)
}
}
// --- decorators -------------------------------------------------------------
// isDecoratorStart reports whether a decorator opens at the scanner's position.
//
// The same rune is Python's matrix-multiplication operator, and the two are
// told apart by where they are: a decorator is the first thing on its line.
func isDecoratorStart(s *syntax.LineScanner) bool {
if s.Peek(0) != '@' || !atLineStart(s) {
return false
}
next := s.Peek(1)
return syntax.IsLetter(next) || next == '_'
}
// takeDecorator colours @property and the dotted name of @app.route.
//
// It stops at the opening parenthesis rather than swallowing to the end of the
// line: the arguments of @pytest.mark.parametrize("n", [1, 2]) are ordinary
// Python, and colouring them as part of the decorator would hide a string and a
// list inside one flat run.
func takeDecorator(s *syntax.LineScanner) {
start := s.Pos()
s.Advance(1) // the @
for !s.AtEnd() && (syntax.IsWordRune(s.Peek(0)) || s.Peek(0) == '.') {
s.Advance(1)
}
s.Emit(start, s.Pos(), syntax.ClassAttribute)
}
// --- where we are on the line -----------------------------------------------
// atLineStart reports whether nothing but indentation comes before the
// scanner's position.
//
// Two decisions need it: a decorator is the first thing on its line, and so is
// the soft keyword that opens a match statement.
func atLineStart(s *syntax.LineScanner) bool {
for at := -1; s.Pos()+at >= 0; at-- {
if r := s.Peek(at); r != ' ' && r != '\t' {
return false
}
}
return true
}
// lineEndsWithColon reports whether the last thing on the line is the colon
// that opens a block.
//
// It reads backwards from the end of the line, which is what makes it cheap
// enough to ask about every word — and also what gives it its one boundary: a
// trailing comment hides the colon from it, so `match x: # dispatch` colours
// match as an identifier. That is the safe direction to be wrong in, and it is
// documented rather than fixed, because telling a real trailing comment from a
// # inside a string means scanning the line forwards, which is the work this
// question is meant to avoid.
func lineEndsWithColon(s *syntax.LineScanner) bool {
for at := s.Len() - s.Pos() - 1; at >= -s.Pos(); at-- {
switch r := s.Peek(at); r {
case ' ', '\t':
case ':':
return true
default:
return false
}
}
return false
}
|