1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
|
package syntax
import (
"path/filepath"
"sort"
"strings"
)
// Language is a file's language, in colouring terms.
//
// It is a string rather than a number because it is written down in places
// outside this package: a snippet restricts itself to `languages = ["go"]`, and
// a language registered by an editor built on this library is not something the
// library could have numbered in advance.
type Language string
// LanguageNone is a file this package cannot colour, which is drawn in plain
// text rather than coloured with rules that do not apply to it.
const LanguageNone Language = ""
// The languages this package colours itself. A language an editor adds — Go,
// Rust — is a constant of that editor's own, not one of these.
const (
// LanguageTOML is a TOML document: theme files, and a project's own
// settings, snippets and tools.
LanguageTOML Language = "toml"
// LanguageMarkdown is a Markdown document, such as this project's own
// documentation.
LanguageMarkdown Language = "markdown"
// LanguageJavaScript is JavaScript, including the module and CommonJS
// spellings of it.
LanguageJavaScript Language = "javascript"
// LanguageHTML is an HTML document.
LanguageHTML Language = "html"
// LanguageBash is a shell script, whether or not it is really bash: the
// keywords and the expansions the scanner knows are the ones sh, bash and
// zsh share.
LanguageBash Language = "bash"
// LanguageDockerfile is a Dockerfile, or a Containerfile, which is the same
// language under another name.
LanguageDockerfile Language = "dockerfile"
// LanguageXML is an XML document, and the many formats that are one under
// another extension: SVG, XSLT, a Maven POM, a .NET project file.
LanguageXML Language = "xml"
// LanguageYAML is a YAML document. A compose file, a Kubernetes manifest
// and a CI workflow are all this: there is no separate dialect, because a
// dialect would be a schema to keep in step with somebody else's product.
LanguageYAML Language = "yaml"
)
// String returns the language's name, and "none" for a file nothing colours.
//
// The name is what a snippets file writes in its languages key, so it is part
// of a file format and not only a label.
func (l Language) String() string {
if l == LanguageNone {
return "none"
}
return string(l)
}
// Definition is how one language is recognised and coloured.
//
// It is what an editor registers to teach this package a language of its own:
// Turbo Go registers Go, Turbo Rust registers Rust, and neither of them has to
// be known here for it to work.
type Definition struct {
// Language is the name the language is known by, and the value LanguageOf
// returns for a file of that kind.
Language Language
// Extensions are the file extensions that identify it, with their dots and
// in lower case: ".go", ".rs". A file's extension always decides when one
// of these matches.
Extensions []string
// Filenames are whole file names that identify the language, for files that
// carry no useful extension: "Dockerfile", "Containerfile".
//
// A file matches when its name equals one of these, or when the part before
// its first dot does — so listing "Dockerfile" also recognises
// "Dockerfile.dev" and "Dockerfile.prod" without naming every variant a
// project might invent. The comparison ignores case, as the extension
// comparison does.
Filenames []string
// Shebangs are interpreter names that identify a file with no useful
// extension, matched against the first line: "sh", "bash", "python3". Most
// languages have none.
Shebangs []string
// Highlight colours a whole document, returning one slice of spans per
// line. It must return exactly as many entries as the source has lines,
// which is what ScanLines and LineIndex both guarantee.
Highlight func(src string) [][]Span
}
// registry is every language this package can colour, by name.
//
// It is package-level state, which is deliberate and is the same shape the
// standard library gives the equivalent problem in image.RegisterFormat: an
// editor registers its language once at start-up, before it opens a file, and
// nothing ever removes one.
var registry = map[Language]Definition{}
// Register teaches this package a language.
//
// Registering a language that is already known replaces it, so an editor can
// override one of the built-ins — a project with its own Markdown dialect, say
// — rather than being stuck with this package's opinion of it. The same goes
// for an extension claimed by two languages: the most recent registration wins,
// because it is the more specific statement of the two.
//
// It is not safe to call from two goroutines at once, and there is no reason
// to: registration belongs in start-up, beside the flags.
//
// syntax.Register(syntax.Definition{
// Language: "rust",
// Extensions: []string{".rs"},
// Highlight: highlightRust,
// })
func Register(d Definition) {
registry[d.Language] = d
}
// Registered returns the languages this package can colour, sorted by name.
//
// It is what a "which languages does this editor know?" message is built from,
// and what a test uses to check that an editor registered what it meant to.
func Registered() []Language {
names := make([]Language, 0, len(registry))
for name := range registry {
names = append(names, name)
}
sort.Slice(names, func(i, j int) bool { return names[i] < names[j] })
return names
}
// LanguageOf returns the language of a file, from its extension, then its name,
// and failing both from its first line.
//
// The **extension** decides whenever there is one a registered language claims.
// Failing that the **name** is tried, which is what colours a `Dockerfile`, a
// file that has no extension to go on. Failing both, a **shebang** naming an
// interpreter decides — which is what colours `configure`, a git hook, or a
// script somebody renamed. Pass "" for firstLine when it is not to hand; the
// other two still work.
//
// A file no language claims gives LanguageNone rather than an error: opening a
// PNG in the editor is not a mistake, it is just not coloured.
//
// syntax.LanguageOf("README.md", "# Title") // markdown
// syntax.LanguageOf("Dockerfile.dev", "") // dockerfile
// syntax.LanguageOf("configure", "#!/bin/sh") // bash
// syntax.LanguageOf("notes.txt", "hello") // none
func LanguageOf(path, firstLine string) Language {
name := filepath.Base(path)
if extension := strings.ToLower(filepath.Ext(name)); extension != "" {
if language, ok := languageByExtension(extension); ok {
return language
}
}
if language, ok := languageByFilename(name); ok {
return language
}
return languageByShebang(firstLine)
}
// languageByFilename returns the language claiming a file's name.
//
// The whole name is tried first and then the part before its first dot, so that
// "Dockerfile" also answers for "Dockerfile.dev" without every variant having
// to be listed. A name that is all extension — ".gitignore" — has an empty stem
// and matches nothing, rather than matching a definition that listed "".
func languageByFilename(name string) (Language, bool) {
lower := strings.ToLower(name)
stem, _, _ := strings.Cut(lower, ".")
found, ok := LanguageNone, false
for _, language := range Registered() {
for _, candidate := range registry[language].Filenames {
candidate = strings.ToLower(candidate)
if candidate == "" {
continue
}
if lower == candidate || (stem != "" && stem == candidate) {
found, ok = language, true
}
}
}
return found, ok
}
// languageByExtension returns the language claiming an extension.
//
// The registry is a map, so its iteration order is random; two languages
// claiming one extension are resolved by name so that the answer is at least
// the same every time rather than different on every run.
func languageByExtension(extension string) (Language, bool) {
found, ok := LanguageNone, false
for _, name := range Registered() {
for _, candidate := range registry[name].Extensions {
if candidate == extension {
found, ok = name, true
}
}
}
return found, ok
}
// languageByShebang returns the language whose interpreter a first line names.
//
// Only an interpreter's own name is looked for, as a path element or as the
// argument to env, so that "#!/usr/bin/env -S bash -e" counts and a script
// merely mentioning bash in a comment does not.
func languageByShebang(firstLine string) Language {
if !strings.HasPrefix(firstLine, "#!") {
return LanguageNone
}
for _, field := range strings.Fields(firstLine) {
name := field[strings.LastIndexByte(field, '/')+1:]
for _, language := range Registered() {
for _, shebang := range registry[language].Shebangs {
if name == shebang {
return language
}
}
}
}
return LanguageNone
}
// init registers the languages this package colours itself.
//
// They are registered rather than special-cased so that there is exactly one
// mechanism: what an editor does to add Rust is what this package does to add
// Markdown, which means the extension point is exercised by every test here.
func init() {
Register(Definition{
Language: LanguageTOML,
Extensions: []string{".toml"},
Highlight: highlightTOML,
})
Register(Definition{
Language: LanguageMarkdown,
Extensions: []string{".md", ".markdown"},
Highlight: highlightMarkdown,
})
Register(Definition{
Language: LanguageJavaScript,
Extensions: []string{".js", ".mjs", ".cjs"},
Highlight: highlightJavaScript,
})
Register(Definition{
Language: LanguageHTML,
Extensions: []string{".html", ".htm"},
Highlight: highlightHTML,
})
Register(Definition{
Language: LanguageDockerfile,
Extensions: []string{".dockerfile", ".containerfile"},
Filenames: []string{"Dockerfile", "Containerfile"},
Highlight: highlightDockerfile,
})
Register(Definition{
Language: LanguageXML,
Extensions: []string{".xml", ".xsd", ".xsl", ".xslt", ".svg", ".plist", ".csproj", ".pom"},
Highlight: highlightXML,
})
Register(Definition{
Language: LanguageYAML,
Extensions: []string{".yaml", ".yml"},
Highlight: highlightYAML,
})
Register(Definition{
Language: LanguageBash,
Extensions: []string{".sh", ".bash", ".zsh"},
Shebangs: []string{"sh", "bash", "zsh", "dash", "ksh"},
Highlight: highlightBash,
})
}
|