| 📦 Turbo Python 6fc62ea k33g 9h ago | 1 | package pythonlang |
| 2 | |
| 3 | import "rickub.com/turbo-editors/turbo-core/syntax" |
| 4 | |
| 5 | // carry is what a line of Python leaves open for the next one. |
| 6 | // |
| 7 | // Only one construct in Python crosses a line break, and it does so in two |
| 8 | // ways. A triple-quoted string runs until the matching three quotes, however |
| 9 | // many lines away that is. A single-quoted one runs on only when the line ends |
| 10 | // with a backslash, which escapes the newline — anything else that reaches the |
| 11 | // end of a line with a quote still open is broken source, and is coloured to |
| 12 | // the end of that line and dropped rather than painting the rest of the file. |
| 13 | // |
| 14 | // Which quote opened it has to be remembered rather than guessed: a literal |
| 15 | // opened with three double quotes and one opened with three apostrophes are |
| 16 | // different strings, and the closer of one appearing inside the other closes |
| 17 | // nothing. (Spelling those triples out in words is deliberate — gofmt rewrites |
| 18 | // a bare run of apostrophes in a doc comment into a typographic quote.) |
| 19 | // |
| 20 | // Rawness is deliberately *not* carried. `r"\""` is a complete string: in a raw |
| 21 | // string the backslash stays in the value, but it still stops the quote after |
| 22 | // it from ending the literal. Termination is therefore the same rule for both, |
| 23 | // and a flag saying otherwise would be a flag nothing reads. |
| 24 | type carry struct { |
| 25 | // open says a string ran past the end of a line. |
| 26 | open bool |
| 27 | // quote is the rune that opened it, ' or ". |
| 28 | quote rune |
| 29 | // triple says it was opened with three of them. |
| 30 | triple bool |
| 31 | } |
| 32 | |
| 33 | // Highlight colours Python source. |
| 34 | // |
| 35 | // It is written against syntax.LineScanner, a line at a time, with the one |
| 36 | // multi-line construct above threaded through carry. Python has no tokeniser in |
| 37 | // the Go standard library the way Go does, so this is a scanner in the same |
| 38 | // style as the ones turbo-core ships for TOML, Markdown and shell. |
| 39 | // |
| 40 | // It is deliberately tolerant of broken input: source under the cursor is |
| 41 | // invalid most of the time it is being typed, and a highlighter that gives up |
| 42 | // is a highlighter that flickers off. |
| 43 | func Highlight(src string) [][]syntax.Span { |
| 44 | return syntax.ScanLines(src, scanLine) |
| 45 | } |
| 46 | |
| 47 | // scanLine colours one line and returns what it leaves open. |
| 48 | func scanLine(line []rune, open carry) ([]syntax.Span, carry) { |
| 49 | s := syntax.NewLineScanner(line) |
| 50 | |
| 51 | // Whatever ran past the end of the previous line is finished first: until |
| 52 | // it closes, nothing on this line is code. |
| 53 | if open.open && !continueString(s, &open) { |
| 54 | return s.Spans(), open |
| 55 | } |
| 56 | |
| 57 | for !s.AtEnd() { |
| 58 | scanToken(s, &open) |
| 59 | } |
| 60 | return s.Spans(), open |
| 61 | } |
| 62 | |
| 63 | // scanToken colours whatever starts at the scanner's position. |
| 64 | func scanToken(s *syntax.LineScanner, open *carry) { |
| 65 | r := s.Peek(0) |
| 66 | |
| 67 | switch { |
| 68 | case r == ' ' || r == '\t': |
| 69 | s.SkipSpaces() |
| 70 | case r == '#': |
| 71 | // Python has no block comment. A run of # lines is a run of comments, |
| 72 | // and a """docstring""" is a string, which is what the language calls |
| 73 | // it and what `help()` reads back. |
| 74 | s.TakeRest(syntax.ClassComment) |
| 75 | case isStringStart(s): |
| 76 | // Before words, because f, r, b and u are letters: without this, |
| 77 | // f"{name}" would be an identifier followed by a string. |
| 78 | takeString(s, open) |
| 79 | case isDecoratorStart(s): |
| 80 | takeDecorator(s) |
| 81 | case syntax.IsDigit(r) || r == '.' && syntax.IsDigit(s.Peek(1)): |
| 82 | // .5 is a float, so a dot with a digit after it starts a number. A dot |
| 83 | // with anything else after it is an attribute access. |
| 84 | takeNumber(s) |
| 85 | case syntax.IsLetter(r) || r == '_': |
| 86 | takeWord(s) |
| 87 | case r == ':' && s.Peek(1) != '=': |
| 88 | // A colon opens a block, separates a dict's key from its value, cuts a |
| 89 | // slice and introduces an annotation: structure in every case, so it |
| 90 | // goes with the brackets and the commas rather than with the |
| 91 | // arithmetic. ":" is an operator rune, so this has to come first — and |
| 92 | // it has to let ":=" through, which really is an operator. |
| 93 | s.Take(1, syntax.ClassPunctuation) |
| 94 | case r == '@': |
| 95 | // Not a decorator, or the case above would have taken it: this is the |
| 96 | // matrix-multiplication operator. |
| 97 | s.Take(1, syntax.ClassOperator) |
| 98 | case r == '\\': |
| 99 | // A backslash at the end of a line joins it to the next one. It is |
| 100 | // structure rather than computation, and colouring it says that the |
| 101 | // line does not end where it looks like it ends. |
| 102 | s.Take(1, syntax.ClassPunctuation) |
| 103 | case syntax.IsOperatorRune(r): |
| 104 | s.TakeWhile(syntax.ClassOperator, syntax.IsOperatorRune) |
| 105 | case syntax.IsPunctuationRune(r): |
| 106 | s.Take(1, syntax.ClassPunctuation) |
| 107 | default: |
| 108 | // A rune nothing here claims — a currency sign in a comment-free line, |
| 109 | // an accented letter in an identifier — is stepped over uncoloured |
| 110 | // rather than guessed at. |
| 111 | s.Advance(1) |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | // --- decorators ------------------------------------------------------------- |
| 116 | |
| 117 | // isDecoratorStart reports whether a decorator opens at the scanner's position. |
| 118 | // |
| 119 | // The same rune is Python's matrix-multiplication operator, and the two are |
| 120 | // told apart by where they are: a decorator is the first thing on its line. |
| 121 | func isDecoratorStart(s *syntax.LineScanner) bool { |
| 122 | if s.Peek(0) != '@' || !atLineStart(s) { |
| 123 | return false |
| 124 | } |
| 125 | next := s.Peek(1) |
| 126 | return syntax.IsLetter(next) || next == '_' |
| 127 | } |
| 128 | |
| 129 | // takeDecorator colours @property and the dotted name of @app.route. |
| 130 | // |
| 131 | // It stops at the opening parenthesis rather than swallowing to the end of the |
| 132 | // line: the arguments of @pytest.mark.parametrize("n", [1, 2]) are ordinary |
| 133 | // Python, and colouring them as part of the decorator would hide a string and a |
| 134 | // list inside one flat run. |
| 135 | func takeDecorator(s *syntax.LineScanner) { |
| 136 | start := s.Pos() |
| 137 | s.Advance(1) // the @ |
| 138 | |
| 139 | for !s.AtEnd() && (syntax.IsWordRune(s.Peek(0)) || s.Peek(0) == '.') { |
| 140 | s.Advance(1) |
| 141 | } |
| 142 | s.Emit(start, s.Pos(), syntax.ClassAttribute) |
| 143 | } |
| 144 | |
| 145 | // --- where we are on the line ----------------------------------------------- |
| 146 | |
| 147 | // atLineStart reports whether nothing but indentation comes before the |
| 148 | // scanner's position. |
| 149 | // |
| 150 | // Two decisions need it: a decorator is the first thing on its line, and so is |
| 151 | // the soft keyword that opens a match statement. |
| 152 | func atLineStart(s *syntax.LineScanner) bool { |
| 153 | for at := -1; s.Pos()+at >= 0; at-- { |
| 154 | if r := s.Peek(at); r != ' ' && r != '\t' { |
| 155 | return false |
| 156 | } |
| 157 | } |
| 158 | return true |
| 159 | } |
| 160 | |
| 161 | // lineEndsWithColon reports whether the last thing on the line is the colon |
| 162 | // that opens a block. |
| 163 | // |
| 164 | // It reads backwards from the end of the line, which is what makes it cheap |
| 165 | // enough to ask about every word — and also what gives it its one boundary: a |
| 166 | // trailing comment hides the colon from it, so `match x: # dispatch` colours |
| 167 | // match as an identifier. That is the safe direction to be wrong in, and it is |
| 168 | // documented rather than fixed, because telling a real trailing comment from a |
| 169 | // # inside a string means scanning the line forwards, which is the work this |
| 170 | // question is meant to avoid. |
| 171 | func lineEndsWithColon(s *syntax.LineScanner) bool { |
| 172 | for at := s.Len() - s.Pos() - 1; at >= -s.Pos(); at-- { |
| 173 | switch r := s.Peek(at); r { |
| 174 | case ' ', '\t': |
| 175 | case ':': |
| 176 | return true |
| 177 | default: |
| 178 | return false |
| 179 | } |
| 180 | } |
| 181 | return false |
| 182 | } |