| 🛟 Updated. 28d5985 k33g 18h ago | 1 | package syntax |
| 2 | |
| 3 | import ( |
| 4 | "path/filepath" |
| 5 | "sort" |
| 6 | "strings" |
| 7 | ) |
| 8 | |
| 9 | // Language is a file's language, in colouring terms. |
| 10 | // |
| 11 | // It is a string rather than a number because it is written down in places |
| 12 | // outside this package: a snippet restricts itself to `languages = ["go"]`, and |
| 13 | // a language registered by an editor built on this library is not something the |
| 14 | // library could have numbered in advance. |
| 15 | type Language string |
| 16 | |
| 17 | // LanguageNone is a file this package cannot colour, which is drawn in plain |
| 18 | // text rather than coloured with rules that do not apply to it. |
| 19 | const LanguageNone Language = "" |
| 20 | |
| 21 | // The languages this package colours itself. A language an editor adds — Go, |
| 22 | // Rust — is a constant of that editor's own, not one of these. |
| 23 | const ( |
| 24 | // LanguageTOML is a TOML document: theme files, and a project's own |
| 25 | // settings, snippets and tools. |
| 26 | LanguageTOML Language = "toml" |
| 27 | // LanguageMarkdown is a Markdown document, such as this project's own |
| 28 | // documentation. |
| 29 | LanguageMarkdown Language = "markdown" |
| 30 | // LanguageJavaScript is JavaScript, including the module and CommonJS |
| 31 | // spellings of it. |
| 32 | LanguageJavaScript Language = "javascript" |
| 33 | // LanguageHTML is an HTML document. |
| 34 | LanguageHTML Language = "html" |
| 35 | // LanguageBash is a shell script, whether or not it is really bash: the |
| 36 | // keywords and the expansions the scanner knows are the ones sh, bash and |
| 37 | // zsh share. |
| 38 | LanguageBash Language = "bash" |
| 39 | // LanguageDockerfile is a Dockerfile, or a Containerfile, which is the same |
| 40 | // language under another name. |
| 41 | LanguageDockerfile Language = "dockerfile" |
| 42 | // LanguageXML is an XML document, and the many formats that are one under |
| 43 | // another extension: SVG, XSLT, a Maven POM, a .NET project file. |
| 44 | LanguageXML Language = "xml" |
| 45 | // LanguageYAML is a YAML document. A compose file, a Kubernetes manifest |
| 46 | // and a CI workflow are all this: there is no separate dialect, because a |
| 47 | // dialect would be a schema to keep in step with somebody else's product. |
| 48 | LanguageYAML Language = "yaml" |
| 49 | ) |
| 50 | |
| 51 | // String returns the language's name, and "none" for a file nothing colours. |
| 52 | // |
| 53 | // The name is what a snippets file writes in its languages key, so it is part |
| 54 | // of a file format and not only a label. |
| 55 | func (l Language) String() string { |
| 56 | if l == LanguageNone { |
| 57 | return "none" |
| 58 | } |
| 59 | return string(l) |
| 60 | } |
| 61 | |
| 62 | // Definition is how one language is recognised and coloured. |
| 63 | // |
| 64 | // It is what an editor registers to teach this package a language of its own: |
| 65 | // Turbo Go registers Go, Turbo Rust registers Rust, and neither of them has to |
| 66 | // be known here for it to work. |
| 67 | type Definition struct { |
| 68 | // Language is the name the language is known by, and the value LanguageOf |
| 69 | // returns for a file of that kind. |
| 70 | Language Language |
| 71 | |
| 72 | // Extensions are the file extensions that identify it, with their dots and |
| 73 | // in lower case: ".go", ".rs". A file's extension always decides when one |
| 74 | // of these matches. |
| 75 | Extensions []string |
| 76 | |
| 77 | // Filenames are whole file names that identify the language, for files that |
| 78 | // carry no useful extension: "Dockerfile", "Containerfile". |
| 79 | // |
| 80 | // A file matches when its name equals one of these, or when the part before |
| 81 | // its first dot does — so listing "Dockerfile" also recognises |
| 82 | // "Dockerfile.dev" and "Dockerfile.prod" without naming every variant a |
| 83 | // project might invent. The comparison ignores case, as the extension |
| 84 | // comparison does. |
| 85 | Filenames []string |
| 86 | |
| 87 | // Shebangs are interpreter names that identify a file with no useful |
| 88 | // extension, matched against the first line: "sh", "bash", "python3". Most |
| 89 | // languages have none. |
| 90 | Shebangs []string |
| 91 | |
| 92 | // Highlight colours a whole document, returning one slice of spans per |
| 93 | // line. It must return exactly as many entries as the source has lines, |
| 94 | // which is what ScanLines and LineIndex both guarantee. |
| 95 | Highlight func(src string) [][]Span |
| 96 | } |
| 97 | |
| 98 | // registry is every language this package can colour, by name. |
| 99 | // |
| 100 | // It is package-level state, which is deliberate and is the same shape the |
| 101 | // standard library gives the equivalent problem in image.RegisterFormat: an |
| 102 | // editor registers its language once at start-up, before it opens a file, and |
| 103 | // nothing ever removes one. |
| 104 | var registry = map[Language]Definition{} |
| 105 | |
| 106 | // Register teaches this package a language. |
| 107 | // |
| 108 | // Registering a language that is already known replaces it, so an editor can |
| 109 | // override one of the built-ins — a project with its own Markdown dialect, say |
| 110 | // — rather than being stuck with this package's opinion of it. The same goes |
| 111 | // for an extension claimed by two languages: the most recent registration wins, |
| 112 | // because it is the more specific statement of the two. |
| 113 | // |
| 114 | // It is not safe to call from two goroutines at once, and there is no reason |
| 115 | // to: registration belongs in start-up, beside the flags. |
| 116 | // |
| 117 | // syntax.Register(syntax.Definition{ |
| 118 | // Language: "rust", |
| 119 | // Extensions: []string{".rs"}, |
| 120 | // Highlight: highlightRust, |
| 121 | // }) |
| 122 | func Register(d Definition) { |
| 123 | registry[d.Language] = d |
| 124 | } |
| 125 | |
| 126 | // Registered returns the languages this package can colour, sorted by name. |
| 127 | // |
| 128 | // It is what a "which languages does this editor know?" message is built from, |
| 129 | // and what a test uses to check that an editor registered what it meant to. |
| 130 | func Registered() []Language { |
| 131 | names := make([]Language, 0, len(registry)) |
| 132 | for name := range registry { |
| 133 | names = append(names, name) |
| 134 | } |
| 135 | sort.Slice(names, func(i, j int) bool { return names[i] < names[j] }) |
| 136 | return names |
| 137 | } |
| 138 | |
| 139 | // LanguageOf returns the language of a file, from its extension, then its name, |
| 140 | // and failing both from its first line. |
| 141 | // |
| 142 | // The **extension** decides whenever there is one a registered language claims. |
| 143 | // Failing that the **name** is tried, which is what colours a `Dockerfile`, a |
| 144 | // file that has no extension to go on. Failing both, a **shebang** naming an |
| 145 | // interpreter decides — which is what colours `configure`, a git hook, or a |
| 146 | // script somebody renamed. Pass "" for firstLine when it is not to hand; the |
| 147 | // other two still work. |
| 148 | // |
| 149 | // A file no language claims gives LanguageNone rather than an error: opening a |
| 150 | // PNG in the editor is not a mistake, it is just not coloured. |
| 151 | // |
| 152 | // syntax.LanguageOf("README.md", "# Title") // markdown |
| 153 | // syntax.LanguageOf("Dockerfile.dev", "") // dockerfile |
| 154 | // syntax.LanguageOf("configure", "#!/bin/sh") // bash |
| 155 | // syntax.LanguageOf("notes.txt", "hello") // none |
| 156 | func LanguageOf(path, firstLine string) Language { |
| 157 | name := filepath.Base(path) |
| 158 | |
| 159 | if extension := strings.ToLower(filepath.Ext(name)); extension != "" { |
| 160 | if language, ok := languageByExtension(extension); ok { |
| 161 | return language |
| 162 | } |
| 163 | } |
| 164 | if language, ok := languageByFilename(name); ok { |
| 165 | return language |
| 166 | } |
| 167 | return languageByShebang(firstLine) |
| 168 | } |
| 169 | |
| 170 | // languageByFilename returns the language claiming a file's name. |
| 171 | // |
| 172 | // The whole name is tried first and then the part before its first dot, so that |
| 173 | // "Dockerfile" also answers for "Dockerfile.dev" without every variant having |
| 174 | // to be listed. A name that is all extension — ".gitignore" — has an empty stem |
| 175 | // and matches nothing, rather than matching a definition that listed "". |
| 176 | func languageByFilename(name string) (Language, bool) { |
| 177 | lower := strings.ToLower(name) |
| 178 | stem, _, _ := strings.Cut(lower, ".") |
| 179 | |
| 180 | found, ok := LanguageNone, false |
| 181 | for _, language := range Registered() { |
| 182 | for _, candidate := range registry[language].Filenames { |
| 183 | candidate = strings.ToLower(candidate) |
| 184 | if candidate == "" { |
| 185 | continue |
| 186 | } |
| 187 | if lower == candidate || (stem != "" && stem == candidate) { |
| 188 | found, ok = language, true |
| 189 | } |
| 190 | } |
| 191 | } |
| 192 | return found, ok |
| 193 | } |
| 194 | |
| 195 | // languageByExtension returns the language claiming an extension. |
| 196 | // |
| 197 | // The registry is a map, so its iteration order is random; two languages |
| 198 | // claiming one extension are resolved by name so that the answer is at least |
| 199 | // the same every time rather than different on every run. |
| 200 | func languageByExtension(extension string) (Language, bool) { |
| 201 | found, ok := LanguageNone, false |
| 202 | for _, name := range Registered() { |
| 203 | for _, candidate := range registry[name].Extensions { |
| 204 | if candidate == extension { |
| 205 | found, ok = name, true |
| 206 | } |
| 207 | } |
| 208 | } |
| 209 | return found, ok |
| 210 | } |
| 211 | |
| 212 | // languageByShebang returns the language whose interpreter a first line names. |
| 213 | // |
| 214 | // Only an interpreter's own name is looked for, as a path element or as the |
| 215 | // argument to env, so that "#!/usr/bin/env -S bash -e" counts and a script |
| 216 | // merely mentioning bash in a comment does not. |
| 217 | func languageByShebang(firstLine string) Language { |
| 218 | if !strings.HasPrefix(firstLine, "#!") { |
| 219 | return LanguageNone |
| 220 | } |
| 221 | |
| 222 | for _, field := range strings.Fields(firstLine) { |
| 223 | name := field[strings.LastIndexByte(field, '/')+1:] |
| 224 | for _, language := range Registered() { |
| 225 | for _, shebang := range registry[language].Shebangs { |
| 226 | if name == shebang { |
| 227 | return language |
| 228 | } |
| 229 | } |
| 230 | } |
| 231 | } |
| 232 | return LanguageNone |
| 233 | } |
| 234 | |
| 235 | // init registers the languages this package colours itself. |
| 236 | // |
| 237 | // They are registered rather than special-cased so that there is exactly one |
| 238 | // mechanism: what an editor does to add Rust is what this package does to add |
| 239 | // Markdown, which means the extension point is exercised by every test here. |
| 240 | func init() { |
| 241 | Register(Definition{ |
| 242 | Language: LanguageTOML, |
| 243 | Extensions: []string{".toml"}, |
| 244 | Highlight: highlightTOML, |
| 245 | }) |
| 246 | Register(Definition{ |
| 247 | Language: LanguageMarkdown, |
| 248 | Extensions: []string{".md", ".markdown"}, |
| 249 | Highlight: highlightMarkdown, |
| 250 | }) |
| 251 | Register(Definition{ |
| 252 | Language: LanguageJavaScript, |
| 253 | Extensions: []string{".js", ".mjs", ".cjs"}, |
| 254 | Highlight: highlightJavaScript, |
| 255 | }) |
| 256 | Register(Definition{ |
| 257 | Language: LanguageHTML, |
| 258 | Extensions: []string{".html", ".htm"}, |
| 259 | Highlight: highlightHTML, |
| 260 | }) |
| 261 | Register(Definition{ |
| 262 | Language: LanguageDockerfile, |
| 263 | Extensions: []string{".dockerfile", ".containerfile"}, |
| 264 | Filenames: []string{"Dockerfile", "Containerfile"}, |
| 265 | Highlight: highlightDockerfile, |
| 266 | }) |
| 267 | Register(Definition{ |
| 268 | Language: LanguageXML, |
| 269 | Extensions: []string{".xml", ".xsd", ".xsl", ".xslt", ".svg", ".plist", ".csproj", ".pom"}, |
| 270 | Highlight: highlightXML, |
| 271 | }) |
| 272 | Register(Definition{ |
| 273 | Language: LanguageYAML, |
| 274 | Extensions: []string{".yaml", ".yml"}, |
| 275 | Highlight: highlightYAML, |
| 276 | }) |
| 277 | Register(Definition{ |
| 278 | Language: LanguageBash, |
| 279 | Extensions: []string{".sh", ".bash", ".zsh"}, |
| 280 | Shebangs: []string{"sh", "bash", "zsh", "dash", "ksh"}, |
| 281 | Highlight: highlightBash, |
| 282 | }) |
| 283 | } |