package syntax import ( "path/filepath" "sort" "strings" ) // Language is a file's language, in colouring terms. // // It is a string rather than a number because it is written down in places // outside this package: a snippet restricts itself to `languages = ["go"]`, and // a language registered by an editor built on this library is not something the // library could have numbered in advance. type Language string // LanguageNone is a file this package cannot colour, which is drawn in plain // text rather than coloured with rules that do not apply to it. const LanguageNone Language = "" // The languages this package colours itself. A language an editor adds — Go, // Rust — is a constant of that editor's own, not one of these. const ( // LanguageTOML is a TOML document: theme files, and a project's own // settings, snippets and tools. LanguageTOML Language = "toml" // LanguageMarkdown is a Markdown document, such as this project's own // documentation. LanguageMarkdown Language = "markdown" // LanguageJavaScript is JavaScript, including the module and CommonJS // spellings of it. LanguageJavaScript Language = "javascript" // LanguageHTML is an HTML document. LanguageHTML Language = "html" // LanguageBash is a shell script, whether or not it is really bash: the // keywords and the expansions the scanner knows are the ones sh, bash and // zsh share. LanguageBash Language = "bash" // LanguageDockerfile is a Dockerfile, or a Containerfile, which is the same // language under another name. LanguageDockerfile Language = "dockerfile" // LanguageXML is an XML document, and the many formats that are one under // another extension: SVG, XSLT, a Maven POM, a .NET project file. LanguageXML Language = "xml" // LanguageYAML is a YAML document. A compose file, a Kubernetes manifest // and a CI workflow are all this: there is no separate dialect, because a // dialect would be a schema to keep in step with somebody else's product. LanguageYAML Language = "yaml" ) // String returns the language's name, and "none" for a file nothing colours. // // The name is what a snippets file writes in its languages key, so it is part // of a file format and not only a label. func (l Language) String() string { if l == LanguageNone { return "none" } return string(l) } // Definition is how one language is recognised and coloured. // // It is what an editor registers to teach this package a language of its own: // Turbo Go registers Go, Turbo Rust registers Rust, and neither of them has to // be known here for it to work. type Definition struct { // Language is the name the language is known by, and the value LanguageOf // returns for a file of that kind. Language Language // Extensions are the file extensions that identify it, with their dots and // in lower case: ".go", ".rs". A file's extension always decides when one // of these matches. Extensions []string // Filenames are whole file names that identify the language, for files that // carry no useful extension: "Dockerfile", "Containerfile". // // A file matches when its name equals one of these, or when the part before // its first dot does — so listing "Dockerfile" also recognises // "Dockerfile.dev" and "Dockerfile.prod" without naming every variant a // project might invent. The comparison ignores case, as the extension // comparison does. Filenames []string // Shebangs are interpreter names that identify a file with no useful // extension, matched against the first line: "sh", "bash", "python3". Most // languages have none. Shebangs []string // Highlight colours a whole document, returning one slice of spans per // line. It must return exactly as many entries as the source has lines, // which is what ScanLines and LineIndex both guarantee. Highlight func(src string) [][]Span } // registry is every language this package can colour, by name. // // It is package-level state, which is deliberate and is the same shape the // standard library gives the equivalent problem in image.RegisterFormat: an // editor registers its language once at start-up, before it opens a file, and // nothing ever removes one. var registry = map[Language]Definition{} // Register teaches this package a language. // // Registering a language that is already known replaces it, so an editor can // override one of the built-ins — a project with its own Markdown dialect, say // — rather than being stuck with this package's opinion of it. The same goes // for an extension claimed by two languages: the most recent registration wins, // because it is the more specific statement of the two. // // It is not safe to call from two goroutines at once, and there is no reason // to: registration belongs in start-up, beside the flags. // // syntax.Register(syntax.Definition{ // Language: "rust", // Extensions: []string{".rs"}, // Highlight: highlightRust, // }) func Register(d Definition) { registry[d.Language] = d } // Registered returns the languages this package can colour, sorted by name. // // It is what a "which languages does this editor know?" message is built from, // and what a test uses to check that an editor registered what it meant to. func Registered() []Language { names := make([]Language, 0, len(registry)) for name := range registry { names = append(names, name) } sort.Slice(names, func(i, j int) bool { return names[i] < names[j] }) return names } // LanguageOf returns the language of a file, from its extension, then its name, // and failing both from its first line. // // The **extension** decides whenever there is one a registered language claims. // Failing that the **name** is tried, which is what colours a `Dockerfile`, a // file that has no extension to go on. Failing both, a **shebang** naming an // interpreter decides — which is what colours `configure`, a git hook, or a // script somebody renamed. Pass "" for firstLine when it is not to hand; the // other two still work. // // A file no language claims gives LanguageNone rather than an error: opening a // PNG in the editor is not a mistake, it is just not coloured. // // syntax.LanguageOf("README.md", "# Title") // markdown // syntax.LanguageOf("Dockerfile.dev", "") // dockerfile // syntax.LanguageOf("configure", "#!/bin/sh") // bash // syntax.LanguageOf("notes.txt", "hello") // none func LanguageOf(path, firstLine string) Language { name := filepath.Base(path) if extension := strings.ToLower(filepath.Ext(name)); extension != "" { if language, ok := languageByExtension(extension); ok { return language } } if language, ok := languageByFilename(name); ok { return language } return languageByShebang(firstLine) } // languageByFilename returns the language claiming a file's name. // // The whole name is tried first and then the part before its first dot, so that // "Dockerfile" also answers for "Dockerfile.dev" without every variant having // to be listed. A name that is all extension — ".gitignore" — has an empty stem // and matches nothing, rather than matching a definition that listed "". func languageByFilename(name string) (Language, bool) { lower := strings.ToLower(name) stem, _, _ := strings.Cut(lower, ".") found, ok := LanguageNone, false for _, language := range Registered() { for _, candidate := range registry[language].Filenames { candidate = strings.ToLower(candidate) if candidate == "" { continue } if lower == candidate || (stem != "" && stem == candidate) { found, ok = language, true } } } return found, ok } // languageByExtension returns the language claiming an extension. // // The registry is a map, so its iteration order is random; two languages // claiming one extension are resolved by name so that the answer is at least // the same every time rather than different on every run. func languageByExtension(extension string) (Language, bool) { found, ok := LanguageNone, false for _, name := range Registered() { for _, candidate := range registry[name].Extensions { if candidate == extension { found, ok = name, true } } } return found, ok } // languageByShebang returns the language whose interpreter a first line names. // // Only an interpreter's own name is looked for, as a path element or as the // argument to env, so that "#!/usr/bin/env -S bash -e" counts and a script // merely mentioning bash in a comment does not. func languageByShebang(firstLine string) Language { if !strings.HasPrefix(firstLine, "#!") { return LanguageNone } for _, field := range strings.Fields(firstLine) { name := field[strings.LastIndexByte(field, '/')+1:] for _, language := range Registered() { for _, shebang := range registry[language].Shebangs { if name == shebang { return language } } } } return LanguageNone } // init registers the languages this package colours itself. // // They are registered rather than special-cased so that there is exactly one // mechanism: what an editor does to add Rust is what this package does to add // Markdown, which means the extension point is exercised by every test here. func init() { Register(Definition{ Language: LanguageTOML, Extensions: []string{".toml"}, Highlight: highlightTOML, }) Register(Definition{ Language: LanguageMarkdown, Extensions: []string{".md", ".markdown"}, Highlight: highlightMarkdown, }) Register(Definition{ Language: LanguageJavaScript, Extensions: []string{".js", ".mjs", ".cjs"}, Highlight: highlightJavaScript, }) Register(Definition{ Language: LanguageHTML, Extensions: []string{".html", ".htm"}, Highlight: highlightHTML, }) Register(Definition{ Language: LanguageDockerfile, Extensions: []string{".dockerfile", ".containerfile"}, Filenames: []string{"Dockerfile", "Containerfile"}, Highlight: highlightDockerfile, }) Register(Definition{ Language: LanguageXML, Extensions: []string{".xml", ".xsd", ".xsl", ".xslt", ".svg", ".plist", ".csproj", ".pom"}, Highlight: highlightXML, }) Register(Definition{ Language: LanguageYAML, Extensions: []string{".yaml", ".yml"}, Highlight: highlightYAML, }) Register(Definition{ Language: LanguageBash, Extensions: []string{".sh", ".bash", ".zsh"}, Shebangs: []string{"sh", "bash", "zsh", "dash", "ksh"}, Highlight: highlightBash, }) }