package syntax import "strings" // LineScanner is the shared machinery of the hand-written scanners: a line in // runes, a position in it, and the spans found so far. // // A language is scanned a line at a time, because a Span may not straddle a // line break and almost every construct fits on one line. What does not — a // fenced code block, an HTML comment, a template literal — is carried across // lines by the scanner that owns it, through the state ScanLines threads. // // Columns are runes throughout. A byte offset would put the spans of a line // with an accent in it out of step with what is drawn. // // It is exported because writing a scanner for a new language is what an editor // built on this library does: Turbo Go's Go scanner and Turbo Rust's Rust // scanner are both written against this type, outside this package. type LineScanner struct { line []rune pos int spans []Span } // NewLineScanner starts a scanner on one line of a document. // // ScanLines is the usual way in and calls this for you; use it directly only // when scanning a single line outside a document. func NewLineScanner(line []rune) *LineScanner { return &LineScanner{line: line} } // Spans returns the spans found so far, which is what a scan function hands // back for its line. func (s *LineScanner) Spans() []Span { return s.spans } // Len returns how many runes the line holds. func (s *LineScanner) Len() int { return len(s.line) } // Pos returns the current position, in runes from the start of the line. func (s *LineScanner) Pos() int { return s.pos } // Advance moves forward n runes without colouring them, stopping at the end of // the line. Text nothing colours is drawn in the editor's plain text style. func (s *LineScanner) Advance(n int) { s.pos = min(s.pos+n, len(s.line)) } // AtEnd reports whether the whole line has been consumed. func (s *LineScanner) AtEnd() bool { return s.pos >= len(s.line) } // Peek returns the rune at an offset from the current position, or 0 when that // is past the end — so a scanner can look ahead without a bounds check. func (s *LineScanner) Peek(offset int) rune { at := s.pos + offset if at < 0 || at >= len(s.line) { return 0 } return s.line[at] } // Emit records a span, dropping the empty ones so that callers never have to // check for them. // // Because empty spans vanish, a scanner must pass the start of a span in // rather than patching it onto the last one afterwards: the last one may not // be the one it thinks. func (s *LineScanner) Emit(start, end int, class Class) { if end > start { s.spans = append(s.spans, Span{Start: start, End: end, Class: class}) } } // Take consumes n runes and colours them, stopping at the end of the line. func (s *LineScanner) Take(n int, class Class) { start := s.pos s.pos = min(s.pos+n, len(s.line)) s.Emit(start, s.pos, class) } // TakeWhile consumes runes for as long as they match, colours them, and // reports whether it consumed any. func (s *LineScanner) TakeWhile(class Class, matches func(rune) bool) bool { start := s.pos for !s.AtEnd() && matches(s.line[s.pos]) { s.pos++ } s.Emit(start, s.pos, class) return s.pos > start } // TakeRest consumes and colours everything left on the line, which is what a // comment running to the end of a line does. func (s *LineScanner) TakeRest(class Class) { s.Emit(s.pos, len(s.line), class) s.pos = len(s.line) } // SkipSpaces steps over blanks without colouring them: whitespace no span // covers is drawn in the editor's plain text style, which is what it should be. func (s *LineScanner) SkipSpaces() { for !s.AtEnd() && (s.line[s.pos] == ' ' || s.line[s.pos] == '\t') { s.pos++ } } // HasPrefix reports whether the line reads want at an offset from the current // position. func (s *LineScanner) HasPrefix(offset int, want string) bool { at := s.pos + offset runes := []rune(want) if at+len(runes) > len(s.line) { return false } for i, r := range runes { if s.line[at+i] != r { return false } } return true } // ScanLines runs a per-line scanner over a whole document, threading whatever // state it carries from one line to the next. // // It is what every hand-written language is built on: the scan function says // what one line holds and what is still open at the end of it, and this turns // that into the one-slice-per-line result Highlight promises. // // State is whatever the language needs to carry across a line break — whether // a block comment is open, which fence a code block started with — and starts // at its zero value on the first line. // // func highlightRust(src string) [][]syntax.Span { // return syntax.ScanLines(src, func(line []rune, inComment bool) ([]syntax.Span, bool) { // s := syntax.NewLineScanner(line) // // ... colour the line ... // return s.Spans(), inComment // }) // } func ScanLines[State any](src string, scan func(line []rune, carry State) ([]Span, State)) [][]Span { lines := splitLines(src) out := make([][]Span, len(lines)) var carry State for i, line := range lines { out[i], carry = scan(line, carry) } return out } // splitLines cuts a document into lines of runes, dropping a carriage return // before each line break so a CRLF file colours the same as an LF one. func splitLines(src string) [][]rune { out := make([][]rune, 0, 1) current := make([]rune, 0, 64) for _, r := range src { if r == '\n' { out = append(out, trimCarriageReturn(current)) current = make([]rune, 0, 64) continue } current = append(current, r) } return append(out, trimCarriageReturn(current)) } // trimCarriageReturn drops a trailing \r, which is what a CRLF line leaves. func trimCarriageReturn(line []rune) []rune { if len(line) > 0 && line[len(line)-1] == '\r' { return line[:len(line)-1] } return line } // TakeQuoted colours a quoted string that ends on its own line, backslash // escapes included. // // A string that is never closed is coloured to the end of the line rather than // abandoned: a string is unterminated most of the time it is being typed, and // giving up would make the colours flicker off at every keystroke. func TakeQuoted(s *LineScanner, quote rune, class Class) { start := s.pos s.pos++ // the opening quote for !s.AtEnd() { if s.line[s.pos] == '\\' && s.pos+1 < len(s.line) { s.pos += 2 continue } if s.line[s.pos] == quote { s.pos++ s.Emit(start, s.pos, class) return } s.pos++ } s.Emit(start, len(s.line), class) } // OpenBlockComment colours a comment that starts at the current position, and // reports whether it also ended on this line. // // A false return is what a scanner carries into the next line as its state. func OpenBlockComment(s *LineScanner, opener, closer string, class Class) bool { start := s.pos s.pos += len([]rune(opener)) if at := indexRunesFrom(s.line, s.pos, closer); at >= 0 { s.pos = at + len([]rune(closer)) s.Emit(start, s.pos, class) return true } s.Emit(start, len(s.line), class) s.pos = len(s.line) return false } // FinishBlockComment colours the continuation of a comment opened on an earlier // line, and reports whether it ended on this one. func FinishBlockComment(s *LineScanner, closer string, class Class) bool { if at := indexRunesFrom(s.line, 0, closer); at >= 0 { s.pos = at + len([]rune(closer)) s.Emit(0, s.pos, class) return true } s.TakeRest(class) return false } // indexRunesFrom returns where want first appears at or after an offset, in // rune columns, or -1. func indexRunesFrom(line []rune, from int, want string) int { runes := []rune(want) for at := max(from, 0); at+len(runes) <= len(line); at++ { if hasRunes(line, at, runes) { return at } } return -1 } // operatorRunes are the characters that make up an operator in the C-like // languages here. They are taken in runs, so that "===" is one span. const operatorRunes = "+-*/%=<>!&|^~?:" // IsOperatorRune reports whether a rune is one of them. func IsOperatorRune(r rune) bool { return strings.ContainsRune(operatorRunes, r) } // punctuationRunes are the characters that structure code rather than compute // with it, split out so a theme can quiet them down. const punctuationRunes = "()[]{},;." // IsPunctuationRune reports whether a rune is one of them. func IsPunctuationRune(r rune) bool { return strings.ContainsRune(punctuationRunes, r) } // IsDigit reports whether a rune is an ASCII digit. func IsDigit(r rune) bool { return r >= '0' && r <= '9' } // IsLetter reports whether a rune is an ASCII letter. func IsLetter(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') } // IsWordRune reports whether a rune can appear in an identifier in most of the // languages here: a letter, a digit, or an underscore. func IsWordRune(r rune) bool { return IsLetter(r) || IsDigit(r) || r == '_' }