| 🛟 Updated. 28d5985 k33g 6h ago | 1 | package syntax |
| 2 | |
| 3 | import "strings" |
| 4 | |
| 5 | // LineScanner is the shared machinery of the hand-written scanners: a line in |
| 6 | // runes, a position in it, and the spans found so far. |
| 7 | // |
| 8 | // A language is scanned a line at a time, because a Span may not straddle a |
| 9 | // line break and almost every construct fits on one line. What does not — a |
| 10 | // fenced code block, an HTML comment, a template literal — is carried across |
| 11 | // lines by the scanner that owns it, through the state ScanLines threads. |
| 12 | // |
| 13 | // Columns are runes throughout. A byte offset would put the spans of a line |
| 14 | // with an accent in it out of step with what is drawn. |
| 15 | // |
| 16 | // It is exported because writing a scanner for a new language is what an editor |
| 17 | // built on this library does: Turbo Go's Go scanner and Turbo Rust's Rust |
| 18 | // scanner are both written against this type, outside this package. |
| 19 | type LineScanner struct { |
| 20 | line []rune |
| 21 | pos int |
| 22 | spans []Span |
| 23 | } |
| 24 | |
| 25 | // NewLineScanner starts a scanner on one line of a document. |
| 26 | // |
| 27 | // ScanLines is the usual way in and calls this for you; use it directly only |
| 28 | // when scanning a single line outside a document. |
| 29 | func NewLineScanner(line []rune) *LineScanner { return &LineScanner{line: line} } |
| 30 | |
| 31 | // Spans returns the spans found so far, which is what a scan function hands |
| 32 | // back for its line. |
| 33 | func (s *LineScanner) Spans() []Span { return s.spans } |
| 34 | |
| 35 | // Len returns how many runes the line holds. |
| 36 | func (s *LineScanner) Len() int { return len(s.line) } |
| 37 | |
| 38 | // Pos returns the current position, in runes from the start of the line. |
| 39 | func (s *LineScanner) Pos() int { return s.pos } |
| 40 | |
| 41 | // Advance moves forward n runes without colouring them, stopping at the end of |
| 42 | // the line. Text nothing colours is drawn in the editor's plain text style. |
| 43 | func (s *LineScanner) Advance(n int) { s.pos = min(s.pos+n, len(s.line)) } |
| 44 | |
| 45 | // AtEnd reports whether the whole line has been consumed. |
| 46 | func (s *LineScanner) AtEnd() bool { return s.pos >= len(s.line) } |
| 47 | |
| 48 | // Peek returns the rune at an offset from the current position, or 0 when that |
| 49 | // is past the end — so a scanner can look ahead without a bounds check. |
| 50 | func (s *LineScanner) Peek(offset int) rune { |
| 51 | at := s.pos + offset |
| 52 | if at < 0 || at >= len(s.line) { |
| 53 | return 0 |
| 54 | } |
| 55 | return s.line[at] |
| 56 | } |
| 57 | |
| 58 | // Emit records a span, dropping the empty ones so that callers never have to |
| 59 | // check for them. |
| 60 | // |
| 61 | // Because empty spans vanish, a scanner must pass the start of a span in |
| 62 | // rather than patching it onto the last one afterwards: the last one may not |
| 63 | // be the one it thinks. |
| 64 | func (s *LineScanner) Emit(start, end int, class Class) { |
| 65 | if end > start { |
| 66 | s.spans = append(s.spans, Span{Start: start, End: end, Class: class}) |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | // Take consumes n runes and colours them, stopping at the end of the line. |
| 71 | func (s *LineScanner) Take(n int, class Class) { |
| 72 | start := s.pos |
| 73 | s.pos = min(s.pos+n, len(s.line)) |
| 74 | s.Emit(start, s.pos, class) |
| 75 | } |
| 76 | |
| 77 | // TakeWhile consumes runes for as long as they match, colours them, and |
| 78 | // reports whether it consumed any. |
| 79 | func (s *LineScanner) TakeWhile(class Class, matches func(rune) bool) bool { |
| 80 | start := s.pos |
| 81 | for !s.AtEnd() && matches(s.line[s.pos]) { |
| 82 | s.pos++ |
| 83 | } |
| 84 | s.Emit(start, s.pos, class) |
| 85 | return s.pos > start |
| 86 | } |
| 87 | |
| 88 | // TakeRest consumes and colours everything left on the line, which is what a |
| 89 | // comment running to the end of a line does. |
| 90 | func (s *LineScanner) TakeRest(class Class) { |
| 91 | s.Emit(s.pos, len(s.line), class) |
| 92 | s.pos = len(s.line) |
| 93 | } |
| 94 | |
| 95 | // SkipSpaces steps over blanks without colouring them: whitespace no span |
| 96 | // covers is drawn in the editor's plain text style, which is what it should be. |
| 97 | func (s *LineScanner) SkipSpaces() { |
| 98 | for !s.AtEnd() && (s.line[s.pos] == ' ' || s.line[s.pos] == '\t') { |
| 99 | s.pos++ |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | // HasPrefix reports whether the line reads want at an offset from the current |
| 104 | // position. |
| 105 | func (s *LineScanner) HasPrefix(offset int, want string) bool { |
| 106 | at := s.pos + offset |
| 107 | runes := []rune(want) |
| 108 | if at+len(runes) > len(s.line) { |
| 109 | return false |
| 110 | } |
| 111 | for i, r := range runes { |
| 112 | if s.line[at+i] != r { |
| 113 | return false |
| 114 | } |
| 115 | } |
| 116 | return true |
| 117 | } |
| 118 | |
| 119 | // ScanLines runs a per-line scanner over a whole document, threading whatever |
| 120 | // state it carries from one line to the next. |
| 121 | // |
| 122 | // It is what every hand-written language is built on: the scan function says |
| 123 | // what one line holds and what is still open at the end of it, and this turns |
| 124 | // that into the one-slice-per-line result Highlight promises. |
| 125 | // |
| 126 | // State is whatever the language needs to carry across a line break — whether |
| 127 | // a block comment is open, which fence a code block started with — and starts |
| 128 | // at its zero value on the first line. |
| 129 | // |
| 130 | // func highlightRust(src string) [][]syntax.Span { |
| 131 | // return syntax.ScanLines(src, func(line []rune, inComment bool) ([]syntax.Span, bool) { |
| 132 | // s := syntax.NewLineScanner(line) |
| 133 | // // ... colour the line ... |
| 134 | // return s.Spans(), inComment |
| 135 | // }) |
| 136 | // } |
| 137 | func ScanLines[State any](src string, scan func(line []rune, carry State) ([]Span, State)) [][]Span { |
| 138 | lines := splitLines(src) |
| 139 | out := make([][]Span, len(lines)) |
| 140 | |
| 141 | var carry State |
| 142 | for i, line := range lines { |
| 143 | out[i], carry = scan(line, carry) |
| 144 | } |
| 145 | return out |
| 146 | } |
| 147 | |
| 148 | // splitLines cuts a document into lines of runes, dropping a carriage return |
| 149 | // before each line break so a CRLF file colours the same as an LF one. |
| 150 | func splitLines(src string) [][]rune { |
| 151 | out := make([][]rune, 0, 1) |
| 152 | current := make([]rune, 0, 64) |
| 153 | |
| 154 | for _, r := range src { |
| 155 | if r == '\n' { |
| 156 | out = append(out, trimCarriageReturn(current)) |
| 157 | current = make([]rune, 0, 64) |
| 158 | continue |
| 159 | } |
| 160 | current = append(current, r) |
| 161 | } |
| 162 | return append(out, trimCarriageReturn(current)) |
| 163 | } |
| 164 | |
| 165 | // trimCarriageReturn drops a trailing \r, which is what a CRLF line leaves. |
| 166 | func trimCarriageReturn(line []rune) []rune { |
| 167 | if len(line) > 0 && line[len(line)-1] == '\r' { |
| 168 | return line[:len(line)-1] |
| 169 | } |
| 170 | return line |
| 171 | } |
| 172 | |
| 173 | // TakeQuoted colours a quoted string that ends on its own line, backslash |
| 174 | // escapes included. |
| 175 | // |
| 176 | // A string that is never closed is coloured to the end of the line rather than |
| 177 | // abandoned: a string is unterminated most of the time it is being typed, and |
| 178 | // giving up would make the colours flicker off at every keystroke. |
| 179 | func TakeQuoted(s *LineScanner, quote rune, class Class) { |
| 180 | start := s.pos |
| 181 | s.pos++ // the opening quote |
| 182 | |
| 183 | for !s.AtEnd() { |
| 184 | if s.line[s.pos] == '\\' && s.pos+1 < len(s.line) { |
| 185 | s.pos += 2 |
| 186 | continue |
| 187 | } |
| 188 | if s.line[s.pos] == quote { |
| 189 | s.pos++ |
| 190 | s.Emit(start, s.pos, class) |
| 191 | return |
| 192 | } |
| 193 | s.pos++ |
| 194 | } |
| 195 | s.Emit(start, len(s.line), class) |
| 196 | } |
| 197 | |
| 198 | // OpenBlockComment colours a comment that starts at the current position, and |
| 199 | // reports whether it also ended on this line. |
| 200 | // |
| 201 | // A false return is what a scanner carries into the next line as its state. |
| 202 | func OpenBlockComment(s *LineScanner, opener, closer string, class Class) bool { |
| 203 | start := s.pos |
| 204 | s.pos += len([]rune(opener)) |
| 205 | |
| 206 | if at := indexRunesFrom(s.line, s.pos, closer); at >= 0 { |
| 207 | s.pos = at + len([]rune(closer)) |
| 208 | s.Emit(start, s.pos, class) |
| 209 | return true |
| 210 | } |
| 211 | s.Emit(start, len(s.line), class) |
| 212 | s.pos = len(s.line) |
| 213 | return false |
| 214 | } |
| 215 | |
| 216 | // FinishBlockComment colours the continuation of a comment opened on an earlier |
| 217 | // line, and reports whether it ended on this one. |
| 218 | func FinishBlockComment(s *LineScanner, closer string, class Class) bool { |
| 219 | if at := indexRunesFrom(s.line, 0, closer); at >= 0 { |
| 220 | s.pos = at + len([]rune(closer)) |
| 221 | s.Emit(0, s.pos, class) |
| 222 | return true |
| 223 | } |
| 224 | s.TakeRest(class) |
| 225 | return false |
| 226 | } |
| 227 | |
| 228 | // indexRunesFrom returns where want first appears at or after an offset, in |
| 229 | // rune columns, or -1. |
| 230 | func indexRunesFrom(line []rune, from int, want string) int { |
| 231 | runes := []rune(want) |
| 232 | for at := max(from, 0); at+len(runes) <= len(line); at++ { |
| 233 | if hasRunes(line, at, runes) { |
| 234 | return at |
| 235 | } |
| 236 | } |
| 237 | return -1 |
| 238 | } |
| 239 | |
| 240 | // operatorRunes are the characters that make up an operator in the C-like |
| 241 | // languages here. They are taken in runs, so that "===" is one span. |
| 242 | const operatorRunes = "+-*/%=<>!&|^~?:" |
| 243 | |
| 244 | // IsOperatorRune reports whether a rune is one of them. |
| 245 | func IsOperatorRune(r rune) bool { return strings.ContainsRune(operatorRunes, r) } |
| 246 | |
| 247 | // punctuationRunes are the characters that structure code rather than compute |
| 248 | // with it, split out so a theme can quiet them down. |
| 249 | const punctuationRunes = "()[]{},;." |
| 250 | |
| 251 | // IsPunctuationRune reports whether a rune is one of them. |
| 252 | func IsPunctuationRune(r rune) bool { return strings.ContainsRune(punctuationRunes, r) } |
| 253 | |
| 254 | // IsDigit reports whether a rune is an ASCII digit. |
| 255 | func IsDigit(r rune) bool { return r >= '0' && r <= '9' } |
| 256 | |
| 257 | // IsLetter reports whether a rune is an ASCII letter. |
| 258 | func IsLetter(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') } |
| 259 | |
| 260 | // IsWordRune reports whether a rune can appear in an identifier in most of the |
| 261 | // languages here: a letter, a digit, or an underscore. |
| 262 | func IsWordRune(r rune) bool { return IsLetter(r) || IsDigit(r) || r == '_' } |