1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
|
package moonbitlang
import "rickub.com/turbo-editors/turbo-core/syntax"
// carry is what a line of MoonBit leaves open for the next one — and in this
// language, nothing does.
//
// Every other editor in this family threads real state through its scanner: Go
// and Rust carry a block-comment depth, Rust carries a raw string's delimiter,
// Python carries which quote opened a triple-quoted literal. MoonBit needs
// none of it, and that is a property of the language rather than a shortcut:
//
// - There is no block comment. The grammar says so in as many words: "MoonBit
// has no block-comment form." // runs to the end of the line, and /// is a
// doc comment that does the same.
// - A newline before a closing quote is an *unterminated literal* error, for
// strings, bytes, regexes, characters and byte characters alike. No literal
// may reach the next line, so none of them can be carried onto it.
// - A multi-line string is not one literal spanning lines. It is a run of
// lines each prefixed #| or $|, each complete in itself, joined afterwards.
// - An attribute is explicitly one line: "everything through the next newline
// is the raw payload".
//
// So the type is empty, and it is a named type rather than struct{} written
// inline so that this comment has somewhere to live. If MoonBit ever grows a
// construct that crosses a line break, this is the type that gains a field and
// scanLine is where it would be threaded.
type carry struct{}
// Highlight colours MoonBit source.
//
// It is written against syntax.LineScanner, a line at a time. MoonBit has no
// tokeniser in the Go standard library the way Go does, so this is a scanner in
// the same style as the ones turbo-core ships for TOML, Markdown and shell.
//
// It is deliberately tolerant of broken input: source under the cursor is
// invalid most of the time it is being typed, and a highlighter that gives up
// is a highlighter that flickers off.
//
// spans := moonbitlang.Highlight("fn main {\n println(\"hi\")\n}\n")
// // spans[0][0] covers "fn" with syntax.ClassKeyword
func Highlight(src string) [][]syntax.Span {
return syntax.ScanLines(src, scanLine)
}
// scanLine colours one line. The carry is threaded because ScanLines asks for
// it, and is returned untouched because nothing in MoonBit crosses a line.
func scanLine(line []rune, open carry) ([]syntax.Span, carry) {
s := syntax.NewLineScanner(line)
for !s.AtEnd() {
scanToken(s)
}
return s.Spans(), open
}
// scanToken colours whatever starts at the scanner's position.
//
// The order of the cases is the design, and three of them are load-bearing:
//
// - "//" is tested before the operators, because / is an operator rune.
// - A multi-line string line is tested before an attribute, because both
// begin with #, and what tells them apart is the rune after it.
// - A literal is tested before a word, because b" and re" begin with letters:
// without this, b"bytes" would be an identifier followed by a string.
func scanToken(s *syntax.LineScanner) {
r := s.Peek(0)
switch {
case r == ' ' || r == '\t':
s.SkipSpaces()
case s.HasPrefix(0, "//"):
// /// is a doc comment and // is an ordinary one. They are the same
// colour because turbo-core's Class set has one comment class, and
// that set is closed on purpose — a theme colours every language an
// editor will ever learn.
s.TakeRest(syntax.ClassComment)
case isMultilineStringStart(s):
takeMultilineStringLine(s)
case isAttributeStart(s):
// The grammar hands the whole rest of the line to the attribute: after
// the dotted name, "everything through the next newline is the raw
// payload". Colouring less than the line would be inventing a
// structure the lexer does not have.
s.TakeRest(syntax.ClassAttribute)
case isPackageStart(s):
takePackageName(s)
case isLiteralStart(s):
takeLiteral(s)
case syntax.IsDigit(r):
// A leading dot is never a number in MoonBit: the grammar requires a
// digit before the point, so .5 is not a literal and .0 is a tuple
// accessor. That is why this case asks for a digit and nothing else.
takeNumber(s)
case r == '.':
takeDot(s)
case syntax.IsLetter(r) || r == '_':
takeWord(s)
case syntax.IsOperatorRune(r):
s.TakeWhile(syntax.ClassOperator, syntax.IsOperatorRune)
case syntax.IsPunctuationRune(r):
s.Take(1, syntax.ClassPunctuation)
default:
// A rune nothing here claims — a CJK letter in an identifier, which
// MoonBit allows and turbo-core's ASCII rune predicates do not — is
// stepped over uncoloured rather than guessed at.
s.Advance(1)
}
}
// --- comments' neighbours: attributes and multi-line strings ----------------
// isAttributeStart reports whether an attribute opens at the scanner's
// position.
//
// An attribute name must begin with a letter or an underscore, which is the
// whole of what separates #deprecated from the #| that opens a raw multi-line
// string line.
func isAttributeStart(s *syntax.LineScanner) bool {
if s.Peek(0) != '#' {
return false
}
next := s.Peek(1)
return syntax.IsLetter(next) || next == '_'
}
// isMultilineStringStart reports whether a multi-line string line opens at the
// scanner's position.
//
// #| is a raw line and $| an interpolated one. Neither is a literal that
// continues: the lines are separate tokens which the compiler joins with a
// newline afterwards, which is exactly why this scanner needs no carry.
func isMultilineStringStart(s *syntax.LineScanner) bool {
return (s.Peek(0) == '#' || s.Peek(0) == '$') && s.Peek(1) == '|'
}
// takeMultilineStringLine colours one #| or $| line.
//
// The two-rune prefix is punctuation rather than string, because it is not part
// of the value: it says "this line is text", and the text is what follows it.
func takeMultilineStringLine(s *syntax.LineScanner) {
s.Take(2, syntax.ClassPunctuation)
s.TakeRest(syntax.ClassString)
}
// --- package names ----------------------------------------------------------
// isPackageStart reports whether a package name opens at the scanner's
// position: @ followed by the first rune of a package part.
func isPackageStart(s *syntax.LineScanner) bool {
if s.Peek(0) != '@' {
return false
}
next := s.Peek(1)
return syntax.IsLetter(next) || next == '_'
}
// takePackageName colours @json, @moonbitlang/core/builtin and the like.
//
// It is one span including the @, because that is one token: the grammar says
// "the leading @, slashes, and parts must be adjacent". ClassType is the
// closest of the seventeen classes — a package qualifier names a thing rather
// than holding a value, and the reading it has to be saved from is the one
// where @json.parse looks like a local variable called json.
//
// A slash or a hyphen is only taken when a name follows it, so @a/2 stops at
// the slash and @a - b stops at the space, rather than swallowing an operator
// because it happened to touch a package name.
func takePackageName(s *syntax.LineScanner) {
start := s.Pos()
s.Advance(1) // the @
for !s.AtEnd() {
switch r := s.Peek(0); {
case syntax.IsWordRune(r):
s.Advance(1)
case r == '/' && (syntax.IsLetter(s.Peek(1)) || s.Peek(1) == '_'):
s.Advance(1)
case r == '-' && syntax.IsWordRune(s.Peek(1)):
s.Advance(1)
default:
s.Emit(start, s.Pos(), syntax.ClassType)
return
}
}
s.Emit(start, s.Pos(), syntax.ClassType)
}
// --- what follows a dot -----------------------------------------------------
// takeDot colours a dot and whatever the language says belongs with it.
//
// Three things begin with one, and telling them apart is what keeps 1..=2 from
// being read as the number 1. followed by =2:
//
// - a run of dots is a range operator — .. ..= ..< ...
// - a dot with digits after it is a tuple accessor, pair.0
// - a dot with a name after it is a field or a method, xs.length()
//
// The third case is the reason this is a function rather than a punctuation
// rune. A dot-identifier "uses the identifier case rules without consulting the
// keyword table, so .if is valid" — so the name after a dot must not be looked
// up among the keywords, and takeMember is the version of takeWord that does
// not.
func takeDot(s *syntax.LineScanner) {
switch next := s.Peek(1); {
case next == '.':
s.TakeWhile(syntax.ClassOperator, func(r rune) bool { return r == '.' })
case syntax.IsDigit(next):
s.Take(1, syntax.ClassPunctuation)
s.TakeWhile(syntax.ClassNumber, syntax.IsDigit)
case syntax.IsLetter(next) || next == '_':
s.Take(1, syntax.ClassPunctuation)
takeMember(s)
default:
s.Take(1, syntax.ClassPunctuation)
}
}
|