1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
package golang
import (
"go/scanner"
"go/token"
"rickub.com/turbo-editors/turbo-core/syntax"
)
// rawToken is one token as go/scanner reports it, reduced to what colouring
// needs: a byte range and the class it belongs to.
type rawToken struct {
start int
end int
tok token.Token
lit string
}
// scanTokens tokenises src, ignoring every syntax error.
//
// Errors are expected: the file is being typed into. The scanner still returns
// a usable token for broken input — an unterminated string comes back as a
// STRING running to the end of the line — which is exactly what keeps the
// colours steady while the user types.
func scanTokens(src string) []rawToken {
fileSet := token.NewFileSet()
file := fileSet.AddFile("", fileSet.Base(), len(src))
var s scanner.Scanner
s.Init(file, []byte(src), func(token.Position, string) {}, scanner.ScanComments)
var tokens []rawToken
for {
pos, tok, lit := s.Scan()
if tok == token.EOF {
return tokens
}
start := file.Offset(pos)
width := tokenWidth(tok, lit)
if width == 0 {
continue // an automatically inserted semicolon covers no text
}
tokens = append(tokens, rawToken{start: start, end: start + width, tok: tok, lit: lit})
}
}
// tokenWidth returns how many bytes of source a token covers.
//
// Operators and punctuation come back with an empty literal, so their width is
// that of their spelling; a semicolon the scanner inserted itself covers
// nothing at all.
func tokenWidth(tok token.Token, lit string) int {
if tok == token.SEMICOLON && lit == "\n" {
return 0
}
if lit != "" {
return len(lit)
}
return len(tok.String())
}
// classify assigns a colouring class to each token, using the token that
// follows and the one before where that is what distinguishes them: an
// identifier before "(" is a call, and one after "type" is a type name.
func classify(tokens []rawToken) []syntax.Class {
classes := make([]syntax.Class, len(tokens))
for i, t := range tokens {
classes[i] = classOf(t, previous(tokens, i), next(tokens, i))
}
return classes
}
// literalClasses are the tokens whose class follows from the token alone.
var literalClasses = map[token.Token]syntax.Class{
token.COMMENT: syntax.ClassComment,
token.STRING: syntax.ClassString,
token.CHAR: syntax.ClassChar,
token.INT: syntax.ClassNumber,
token.FLOAT: syntax.ClassNumber,
token.IMAG: syntax.ClassNumber,
}
// classOf returns the class of a single token, given its neighbours.
func classOf(t rawToken, before, after token.Token) syntax.Class {
if class, ok := literalClasses[t.tok]; ok {
return class
}
if t.tok == token.IDENT {
return identifierClass(t.lit, before, after)
}
return symbolClass(t.tok)
}
// symbolClass returns the class of anything that is neither a literal nor an
// identifier: a keyword, a bracket, or an operator.
func symbolClass(tok token.Token) syntax.Class {
switch {
case tok.IsKeyword():
return syntax.ClassKeyword
case isPunctuation(tok):
return syntax.ClassPunctuation
case tok.IsOperator():
return syntax.ClassOperator
}
return syntax.ClassIdentifier
}
// predeclaredClasses are the identifiers the language itself provides, and
// what each of them is. They are recognised by name because they are not
// keywords: a file may shadow "len" or "any", and colouring it as the
// predeclared one anyway is what every other Go editor does too.
var predeclaredClasses = map[string]syntax.Class{
// Types.
"any": syntax.ClassType, "bool": syntax.ClassType, "byte": syntax.ClassType,
"comparable": syntax.ClassType, "complex64": syntax.ClassType, "complex128": syntax.ClassType,
"error": syntax.ClassType, "float32": syntax.ClassType, "float64": syntax.ClassType,
"int": syntax.ClassType, "int8": syntax.ClassType, "int16": syntax.ClassType,
"int32": syntax.ClassType, "int64": syntax.ClassType, "rune": syntax.ClassType,
"string": syntax.ClassType, "uint": syntax.ClassType, "uint8": syntax.ClassType,
"uint16": syntax.ClassType, "uint32": syntax.ClassType, "uint64": syntax.ClassType,
"uintptr": syntax.ClassType,
// Constants.
"true": syntax.ClassConstant, "false": syntax.ClassConstant,
"iota": syntax.ClassConstant, "nil": syntax.ClassConstant,
// Functions.
"append": syntax.ClassBuiltin, "cap": syntax.ClassBuiltin, "clear": syntax.ClassBuiltin,
"close": syntax.ClassBuiltin, "complex": syntax.ClassBuiltin, "copy": syntax.ClassBuiltin,
"delete": syntax.ClassBuiltin, "imag": syntax.ClassBuiltin, "len": syntax.ClassBuiltin,
"make": syntax.ClassBuiltin, "max": syntax.ClassBuiltin, "min": syntax.ClassBuiltin,
"new": syntax.ClassBuiltin, "panic": syntax.ClassBuiltin, "print": syntax.ClassBuiltin,
"println": syntax.ClassBuiltin, "real": syntax.ClassBuiltin, "recover": syntax.ClassBuiltin,
}
// identifierClass tells apart the several things an identifier can be.
func identifierClass(name string, before, after token.Token) syntax.Class {
if class, ok := predeclaredClasses[name]; ok {
return class
}
class := syntax.ClassIdentifier
switch {
case before == token.TYPE, before == token.STRUCT, before == token.INTERFACE:
class = syntax.ClassType
case after == token.LPAREN, before == token.FUNC:
class = syntax.ClassFunction
}
return class
}
// previous returns the token before index i, or ILLEGAL at the start.
func previous(tokens []rawToken, i int) token.Token {
if i == 0 {
return token.ILLEGAL
}
return tokens[i-1].tok
}
// next returns the token after index i, or ILLEGAL at the end.
func next(tokens []rawToken, i int) token.Token {
if i+1 >= len(tokens) {
return token.ILLEGAL
}
return tokens[i+1].tok
}
// isPunctuation reports whether a token is structure rather than computation.
// Brackets, commas and the like are usually themed more quietly than the
// operators that actually do something.
func isPunctuation(tok token.Token) bool {
switch tok {
case token.LPAREN, token.RPAREN,
token.LBRACK, token.RBRACK,
token.LBRACE, token.RBRACE,
token.COMMA, token.SEMICOLON, token.COLON, token.PERIOD:
return true
default:
return false
}
}
|