1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262

package syntax

import "strings"

// LineScanner is the shared machinery of the hand-written scanners: a line in
// runes, a position in it, and the spans found so far.
//
// A language is scanned a line at a time, because a Span may not straddle a
// line break and almost every construct fits on one line. What does not — a
// fenced code block, an HTML comment, a template literal — is carried across
// lines by the scanner that owns it, through the state ScanLines threads.
//
// Columns are runes throughout. A byte offset would put the spans of a line
// with an accent in it out of step with what is drawn.
//
// It is exported because writing a scanner for a new language is what an editor
// built on this library does: Turbo Go's Go scanner and Turbo Rust's Rust
// scanner are both written against this type, outside this package.
type LineScanner struct {
	line  []rune
	pos   int
	spans []Span
}

// NewLineScanner starts a scanner on one line of a document.
//
// ScanLines is the usual way in and calls this for you; use it directly only
// when scanning a single line outside a document.
func NewLineScanner(line []rune) *LineScanner { return &LineScanner{line: line} }

// Spans returns the spans found so far, which is what a scan function hands
// back for its line.
func (s *LineScanner) Spans() []Span { return s.spans }

// Len returns how many runes the line holds.
func (s *LineScanner) Len() int { return len(s.line) }

// Pos returns the current position, in runes from the start of the line.
func (s *LineScanner) Pos() int { return s.pos }

// Advance moves forward n runes without colouring them, stopping at the end of
// the line. Text nothing colours is drawn in the editor's plain text style.
func (s *LineScanner) Advance(n int) { s.pos = min(s.pos+n, len(s.line)) }

// AtEnd reports whether the whole line has been consumed.
func (s *LineScanner) AtEnd() bool { return s.pos >= len(s.line) }

// Peek returns the rune at an offset from the current position, or 0 when that
// is past the end — so a scanner can look ahead without a bounds check.
func (s *LineScanner) Peek(offset int) rune {
	at := s.pos + offset
	if at < 0 || at >= len(s.line) {
		return 0
	}
	return s.line[at]
}

// Emit records a span, dropping the empty ones so that callers never have to
// check for them.
//
// Because empty spans vanish, a scanner must pass the start of a span in
// rather than patching it onto the last one afterwards: the last one may not
// be the one it thinks.
func (s *LineScanner) Emit(start, end int, class Class) {
	if end > start {
		s.spans = append(s.spans, Span{Start: start, End: end, Class: class})
	}
}

// Take consumes n runes and colours them, stopping at the end of the line.
func (s *LineScanner) Take(n int, class Class) {
	start := s.pos
	s.pos = min(s.pos+n, len(s.line))
	s.Emit(start, s.pos, class)
}

// TakeWhile consumes runes for as long as they match, colours them, and
// reports whether it consumed any.
func (s *LineScanner) TakeWhile(class Class, matches func(rune) bool) bool {
	start := s.pos
	for !s.AtEnd() && matches(s.line[s.pos]) {
		s.pos++
	}
	s.Emit(start, s.pos, class)
	return s.pos > start
}

// TakeRest consumes and colours everything left on the line, which is what a
// comment running to the end of a line does.
func (s *LineScanner) TakeRest(class Class) {
	s.Emit(s.pos, len(s.line), class)
	s.pos = len(s.line)
}

// SkipSpaces steps over blanks without colouring them: whitespace no span
// covers is drawn in the editor's plain text style, which is what it should be.
func (s *LineScanner) SkipSpaces() {
	for !s.AtEnd() && (s.line[s.pos] == ' ' || s.line[s.pos] == '\t') {
		s.pos++
	}
}

// HasPrefix reports whether the line reads want at an offset from the current
// position.
func (s *LineScanner) HasPrefix(offset int, want string) bool {
	at := s.pos + offset
	runes := []rune(want)
	if at+len(runes) > len(s.line) {
		return false
	}
	for i, r := range runes {
		if s.line[at+i] != r {
			return false
		}
	}
	return true
}

// ScanLines runs a per-line scanner over a whole document, threading whatever
// state it carries from one line to the next.
//
// It is what every hand-written language is built on: the scan function says
// what one line holds and what is still open at the end of it, and this turns
// that into the one-slice-per-line result Highlight promises.
//
// State is whatever the language needs to carry across a line break — whether
// a block comment is open, which fence a code block started with — and starts
// at its zero value on the first line.
//
//	func highlightRust(src string) [][]syntax.Span {
//		return syntax.ScanLines(src, func(line []rune, inComment bool) ([]syntax.Span, bool) {
//			s := syntax.NewLineScanner(line)
//			// ... colour the line ...
//			return s.Spans(), inComment
//		})
//	}
func ScanLines[State any](src string, scan func(line []rune, carry State) ([]Span, State)) [][]Span {
	lines := splitLines(src)
	out := make([][]Span, len(lines))

	var carry State
	for i, line := range lines {
		out[i], carry = scan(line, carry)
	}
	return out
}

// splitLines cuts a document into lines of runes, dropping a carriage return
// before each line break so a CRLF file colours the same as an LF one.
func splitLines(src string) [][]rune {
	out := make([][]rune, 0, 1)
	current := make([]rune, 0, 64)

	for _, r := range src {
		if r == '\n' {
			out = append(out, trimCarriageReturn(current))
			current = make([]rune, 0, 64)
			continue
		}
		current = append(current, r)
	}
	return append(out, trimCarriageReturn(current))
}

// trimCarriageReturn drops a trailing \r, which is what a CRLF line leaves.
func trimCarriageReturn(line []rune) []rune {
	if len(line) > 0 && line[len(line)-1] == '\r' {
		return line[:len(line)-1]
	}
	return line
}

// TakeQuoted colours a quoted string that ends on its own line, backslash
// escapes included.
//
// A string that is never closed is coloured to the end of the line rather than
// abandoned: a string is unterminated most of the time it is being typed, and
// giving up would make the colours flicker off at every keystroke.
func TakeQuoted(s *LineScanner, quote rune, class Class) {
	start := s.pos
	s.pos++ // the opening quote

	for !s.AtEnd() {
		if s.line[s.pos] == '\\' && s.pos+1 < len(s.line) {
			s.pos += 2
			continue
		}
		if s.line[s.pos] == quote {
			s.pos++
			s.Emit(start, s.pos, class)
			return
		}
		s.pos++
	}
	s.Emit(start, len(s.line), class)
}

// OpenBlockComment colours a comment that starts at the current position, and
// reports whether it also ended on this line.
//
// A false return is what a scanner carries into the next line as its state.
func OpenBlockComment(s *LineScanner, opener, closer string, class Class) bool {
	start := s.pos
	s.pos += len([]rune(opener))

	if at := indexRunesFrom(s.line, s.pos, closer); at >= 0 {
		s.pos = at + len([]rune(closer))
		s.Emit(start, s.pos, class)
		return true
	}
	s.Emit(start, len(s.line), class)
	s.pos = len(s.line)
	return false
}

// FinishBlockComment colours the continuation of a comment opened on an earlier
// line, and reports whether it ended on this one.
func FinishBlockComment(s *LineScanner, closer string, class Class) bool {
	if at := indexRunesFrom(s.line, 0, closer); at >= 0 {
		s.pos = at + len([]rune(closer))
		s.Emit(0, s.pos, class)
		return true
	}
	s.TakeRest(class)
	return false
}

// indexRunesFrom returns where want first appears at or after an offset, in
// rune columns, or -1.
func indexRunesFrom(line []rune, from int, want string) int {
	runes := []rune(want)
	for at := max(from, 0); at+len(runes) <= len(line); at++ {
		if hasRunes(line, at, runes) {
			return at
		}
	}
	return -1
}

// operatorRunes are the characters that make up an operator in the C-like
// languages here. They are taken in runs, so that "===" is one span.
const operatorRunes = "+-*/%=<>!&|^~?:"

// IsOperatorRune reports whether a rune is one of them.
func IsOperatorRune(r rune) bool { return strings.ContainsRune(operatorRunes, r) }

// punctuationRunes are the characters that structure code rather than compute
// with it, split out so a theme can quiet them down.
const punctuationRunes = "()[]{},;."

// IsPunctuationRune reports whether a rune is one of them.
func IsPunctuationRune(r rune) bool { return strings.ContainsRune(punctuationRunes, r) }

// IsDigit reports whether a rune is an ASCII digit.
func IsDigit(r rune) bool { return r >= '0' && r <= '9' }

// IsLetter reports whether a rune is an ASCII letter.
func IsLetter(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') }

// IsWordRune reports whether a rune can appear in an identifier in most of the
// languages here: a letter, a digit, or an underscore.
func IsWordRune(r rune) bool { return IsLetter(r) || IsDigit(r) || r == '_' }