1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
package rustlang
import "rickub.com/turbo-editors/turbo-core/syntax"
// carry is what a line of Rust leaves open for the next one.
//
// Three constructs in Rust can cross a line break, and each of them has to be
// remembered exactly rather than guessed at: a block comment (which nests, so a
// depth and not a flag), a raw string (whose terminator is a quote followed by
// as many hashes as its opener had), and an ordinary string (which may contain
// a real newline). Anything else is decided from the line in front of you.
type carry struct {
// commentDepth is how many /* are still open. Rust nests block comments,
// so /* /* */ */ is one comment and a flag would end it one level early.
commentDepth int
// rawHashes is the number of # a raw string was opened with, and rawOpen
// says whether one is open at all — a raw string may be opened with none,
// so the count alone cannot say.
rawOpen bool
rawHashes int
// stringOpen says an ordinary "…" string ran past the end of a line.
stringOpen bool
}
// Highlight colours Rust source.
//
// It is written against syntax.LineScanner, a line at a time, with the three
// multi-line constructs above threaded through carry. Rust has no tokeniser in
// the Go standard library the way Go does, so this is a scanner in the same
// style as the ones turbo-core ships for TOML, Markdown and shell.
//
// It is deliberately tolerant of broken input: source under the cursor is
// invalid most of the time it is being typed, and a highlighter that gives up
// is a highlighter that flickers off.
func Highlight(src string) [][]syntax.Span {
return syntax.ScanLines(src, scanLine)
}
// scanLine colours one line and returns what it leaves open.
func scanLine(line []rune, open carry) ([]syntax.Span, carry) {
s := syntax.NewLineScanner(line)
// Whatever ran past the end of the previous line is finished first: until
// it closes, nothing on this line is code.
if !finishCarried(s, &open) {
return s.Spans(), open
}
for !s.AtEnd() {
scanToken(s, &open)
}
return s.Spans(), open
}
// finishCarried closes whatever the previous line left open, and reports
// whether the rest of this line is code.
func finishCarried(s *syntax.LineScanner, open *carry) bool {
switch {
case open.commentDepth > 0:
return continueBlockComment(s, open)
case open.rawOpen:
return continueRawString(s, open)
case open.stringOpen:
return continueString(s, open)
}
return true
}
// scanToken colours whatever starts at the scanner's position.
func scanToken(s *syntax.LineScanner, open *carry) {
r := s.Peek(0)
switch {
case r == ' ' || r == '\t':
s.SkipSpaces()
case s.HasPrefix(0, "//"):
s.TakeRest(syntax.ClassComment)
case s.HasPrefix(0, "/*"):
startBlockComment(s, open)
case r == '#' && (s.Peek(1) == '[' || (s.Peek(1) == '!' && s.Peek(2) == '[')):
takeAttribute(s)
case isRawStringStart(s):
startRawString(s, open)
case isByteOrCharStart(s):
takeByteLiteral(s, open)
case r == '"':
startString(s, open)
case r == '\'':
takeQuoteOrLifetime(s)
case syntax.IsDigit(r):
takeNumber(s)
case syntax.IsLetter(r) || r == '_':
takeWord(s)
case r == ':':
// A path separator and a type annotation are both structure rather
// than computation, so they go with the brackets and the commas. ":"
// is an operator rune, so this has to come first.
s.TakeWhile(syntax.ClassPunctuation, func(c rune) bool { return c == ':' })
case s.HasPrefix(0, ".."):
// A range really is an operation, and "." is a punctuation rune, so
// this has to come first too.
s.Take(rangeWidth(s), syntax.ClassOperator)
case syntax.IsOperatorRune(r):
s.TakeWhile(syntax.ClassOperator, syntax.IsOperatorRune)
case syntax.IsPunctuationRune(r) || r == '#' || r == '@' || r == '$':
s.Take(1, syntax.ClassPunctuation)
default:
// A rune nothing here claims — an emoji in an identifier, say — is
// stepped over uncoloured rather than guessed at.
s.Advance(1)
}
}
// --- comments ---------------------------------------------------------------
// startBlockComment colours a /* that opens on this line, counting the nesting
// Rust allows.
func startBlockComment(s *syntax.LineScanner, open *carry) {
start := s.Pos()
s.Advance(2)
open.commentDepth = 1
consumeComment(s, open)
s.Emit(start, s.Pos(), syntax.ClassComment)
}
// continueBlockComment colours the rest of a comment opened on an earlier line,
// and reports whether the line has code after it.
func continueBlockComment(s *syntax.LineScanner, open *carry) bool {
consumeComment(s, open)
s.Emit(0, s.Pos(), syntax.ClassComment)
return open.commentDepth == 0 && !s.AtEnd()
}
// consumeComment runs to the end of the comment or to the end of the line,
// keeping the nesting depth in step.
func consumeComment(s *syntax.LineScanner, open *carry) {
for !s.AtEnd() {
switch {
case s.HasPrefix(0, "/*"):
open.commentDepth++
s.Advance(2)
case s.HasPrefix(0, "*/"):
open.commentDepth--
s.Advance(2)
if open.commentDepth == 0 {
return
}
default:
s.Advance(1)
}
}
}
// --- attributes -------------------------------------------------------------
// takeAttribute colours #[derive(Debug)] and its inner form #![no_std].
//
// An attribute that runs past the end of its line is coloured to the end and
// not carried: unlike a comment or a string, an unclosed attribute is nearly
// always a half-typed one, and carrying it would paint the rest of the file.
func takeAttribute(s *syntax.LineScanner) {
start := s.Pos()
// Step over the # and the ! of the inner form, so that the bracket
// matching below starts where the brackets actually are. Counting from the
// # instead ends #![no_std] at its second rune, which is a bug this had.
s.Advance(1)
if s.Peek(0) == '!' {
s.Advance(1)
}
depth := 0
for !s.AtEnd() {
switch s.Peek(0) {
case '[':
depth++
case ']':
depth--
}
s.Advance(1)
if depth == 0 {
break
}
}
s.Emit(start, s.Pos(), syntax.ClassAttribute)
}
|