1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
|
package gololang
import "rickub.com/turbo-editors/turbo-core/syntax"
// carry is what a line of Golo leaves open for the next one: which construct,
// if any, the line ended inside.
//
// Four constructs may reach the next line, and the interpreter's own lexer is
// the authority on each. A block comment runs from one ---- to the next
// wherever that is. A "string", a """multi-line string""" and a 'character'
// are all read to their closing quote, and that quote may be on a later line:
// lexer.go reads a plain string with `for l.ch != '"' && l.ch != 0`, which
// stops at the quote or the end of the file and at nothing in between. So an
// unterminated string paints the rest of the file — and that is what the
// interpreter does with it too, which is the reason to carry rather than to
// stop at the line the way Turbo MoonBit does for a language whose grammar
// forbids the newline.
//
// None of the four nests, so a value saying which one is open is the whole of
// the state. A depth would be a claim the language does not make.
type carry int
const (
// nothingOpen is the state between constructs, and the zero value
// ScanLines starts the first line in.
nothingOpen carry = iota
// blockCommentOpen means a ---- was seen and its closing ---- was not.
blockCommentOpen
// stringOpen means a " was seen and its closing " was not.
stringOpen
// multilineStringOpen means a """ was seen and its closing """ was not.
multilineStringOpen
// charOpen means a ' was seen and its closing ' was not.
charOpen
)
// Highlight colours Golo source.
//
// It is written against syntax.LineScanner, a line at a time, with the
// interpreter's lexer/lexer.go as its specification. GoloScript's own lexer is
// a Go package, but its module is named `golo` and is not importable from
// another module, so the rules are carried here rather than called — and what
// the lexer reads as one token, this scanner colours as one span.
//
// It is deliberately tolerant of broken input: source under the cursor is
// invalid most of the time it is being typed, and a highlighter that gives up
// is a highlighter that flickers off.
//
// spans := gololang.Highlight("function main = |args| {\n println(\"hi\")\n}\n")
// // spans[0][0] covers "function" with syntax.ClassKeyword
func Highlight(src string) [][]syntax.Span {
return syntax.ScanLines(src, scanLine)
}
// scanLine colours one line, finishing whatever the previous line left open
// before looking at anything new, and reports what this line leaves open.
func scanLine(line []rune, open carry) ([]syntax.Span, carry) {
s := syntax.NewLineScanner(line)
open = finishOpen(s, open)
for !s.AtEnd() {
open = scanToken(s)
}
return s.Spans(), open
}
// finishOpen colours the continuation of a construct opened on an earlier
// line, and says whether it is still open at the end of this one. With nothing
// open it does nothing.
func finishOpen(s *syntax.LineScanner, open carry) carry {
switch open {
case blockCommentOpen:
return stillOpen(syntax.FinishBlockComment(s, blockCommentMarker, syntax.ClassComment), open)
case multilineStringOpen:
return stillOpen(syntax.FinishBlockComment(s, multilineStringQuote, syntax.ClassString), open)
case stringOpen:
return finishQuoted(s, '"', syntax.ClassString, open)
case charOpen:
return finishQuoted(s, '\'', syntax.ClassChar, open)
default:
return nothingOpen
}
}
// stillOpen turns "did it close?" into what the next line should be told.
func stillOpen(closed bool, open carry) carry {
if closed {
return nothingOpen
}
return open
}
// blockCommentMarker opens and closes a block comment. The lexer asks for four
// dashes exactly: three are an operator run, and a fifth is part of the text.
const blockCommentMarker = "----"
// scanToken colours whatever starts at the scanner's position, and reports
// which construct, if any, ran off the end of the line.
//
// The order of the cases is the design, and three of them are load-bearing:
//
// - ---- is tested before the operators, because - is an operator rune and
// the run would otherwise be coloured as one.
// - """ is tested before ", because both begin with a quote and what tells
// them apart is the two runes after it. The lexer makes the same check in
// the same order.
// - A digit is tested before a word, and a word before a dot, so that 1.5 is
// one number and hello.World is a name, a dot and a name.
func scanToken(s *syntax.LineScanner) carry {
r := s.Peek(0)
switch {
case r == ' ' || r == '\t':
s.SkipSpaces()
case r == '#':
// # opens a comment that runs to the end of the line. A shebang is
// one too: #!/usr/bin/env golo is how a script is run as a command,
// and the interpreter reads it as a comment because that is what it
// is.
s.TakeRest(syntax.ClassComment)
case s.HasPrefix(0, blockCommentMarker):
return stillOpen(syntax.OpenBlockComment(s, blockCommentMarker, blockCommentMarker, syntax.ClassComment), blockCommentOpen)
case s.HasPrefix(0, multilineStringQuote):
return takeMultilineString(s)
case r == '"':
return takeQuoted(s, '"', syntax.ClassString, stringOpen)
case r == '\'':
return takeQuoted(s, '\'', syntax.ClassChar, charOpen)
case syntax.IsDigit(r):
takeNumber(s)
case isIdentifierStart(r):
takeWord(s)
case r == '.':
takeDot(s)
case r == '$':
// $ joins a union to one of its variants in `augment Shape$Circle`.
// It is structure rather than computation, so it is punctuation like
// the dot in a module path.
s.Take(1, syntax.ClassPunctuation)
case syntax.IsOperatorRune(r):
s.TakeWhile(syntax.ClassOperator, syntax.IsOperatorRune)
case syntax.IsPunctuationRune(r):
s.Take(1, syntax.ClassPunctuation)
default:
// A rune nothing here claims — a stray control character, a symbol
// outside the ranges the lexer accepts in a name — is stepped over
// uncoloured rather than guessed at.
s.Advance(1)
}
return nothingOpen
}
// takeDot colours a dot and whatever the language says belongs with it.
//
// A run of dots is an operator — .. is a range and ... marks a variadic
// parameter — and a dot on its own is punctuation: the separator in a module
// path such as hello.World, or in a decimal that takeNumber did not claim
// because no digit came before it.
func takeDot(s *syntax.LineScanner) {
if s.Peek(1) == '.' {
s.TakeWhile(syntax.ClassOperator, func(r rune) bool { return r == '.' })
return
}
s.Take(1, syntax.ClassPunctuation)
}
|