package pythonlang import "rickub.com/turbo-editors/turbo-core/syntax" // carry is what a line of Python leaves open for the next one. // // Only one construct in Python crosses a line break, and it does so in two // ways. A triple-quoted string runs until the matching three quotes, however // many lines away that is. A single-quoted one runs on only when the line ends // with a backslash, which escapes the newline — anything else that reaches the // end of a line with a quote still open is broken source, and is coloured to // the end of that line and dropped rather than painting the rest of the file. // // Which quote opened it has to be remembered rather than guessed: a literal // opened with three double quotes and one opened with three apostrophes are // different strings, and the closer of one appearing inside the other closes // nothing. (Spelling those triples out in words is deliberate — gofmt rewrites // a bare run of apostrophes in a doc comment into a typographic quote.) // // Rawness is deliberately *not* carried. `r"\""` is a complete string: in a raw // string the backslash stays in the value, but it still stops the quote after // it from ending the literal. Termination is therefore the same rule for both, // and a flag saying otherwise would be a flag nothing reads. type carry struct { // open says a string ran past the end of a line. open bool // quote is the rune that opened it, ' or ". quote rune // triple says it was opened with three of them. triple bool } // Highlight colours Python source. // // It is written against syntax.LineScanner, a line at a time, with the one // multi-line construct above threaded through carry. Python has no tokeniser in // the Go standard library the way Go does, so this is a scanner in the same // style as the ones turbo-core ships for TOML, Markdown and shell. // // It is deliberately tolerant of broken input: source under the cursor is // invalid most of the time it is being typed, and a highlighter that gives up // is a highlighter that flickers off. func Highlight(src string) [][]syntax.Span { return syntax.ScanLines(src, scanLine) } // scanLine colours one line and returns what it leaves open. func scanLine(line []rune, open carry) ([]syntax.Span, carry) { s := syntax.NewLineScanner(line) // Whatever ran past the end of the previous line is finished first: until // it closes, nothing on this line is code. if open.open && !continueString(s, &open) { return s.Spans(), open } for !s.AtEnd() { scanToken(s, &open) } return s.Spans(), open } // scanToken colours whatever starts at the scanner's position. func scanToken(s *syntax.LineScanner, open *carry) { r := s.Peek(0) switch { case r == ' ' || r == '\t': s.SkipSpaces() case r == '#': // Python has no block comment. A run of # lines is a run of comments, // and a """docstring""" is a string, which is what the language calls // it and what `help()` reads back. s.TakeRest(syntax.ClassComment) case isStringStart(s): // Before words, because f, r, b and u are letters: without this, // f"{name}" would be an identifier followed by a string. takeString(s, open) case isDecoratorStart(s): takeDecorator(s) case syntax.IsDigit(r) || r == '.' && syntax.IsDigit(s.Peek(1)): // .5 is a float, so a dot with a digit after it starts a number. A dot // with anything else after it is an attribute access. takeNumber(s) case syntax.IsLetter(r) || r == '_': takeWord(s) case r == ':' && s.Peek(1) != '=': // A colon opens a block, separates a dict's key from its value, cuts a // slice and introduces an annotation: structure in every case, so it // goes with the brackets and the commas rather than with the // arithmetic. ":" is an operator rune, so this has to come first — and // it has to let ":=" through, which really is an operator. s.Take(1, syntax.ClassPunctuation) case r == '@': // Not a decorator, or the case above would have taken it: this is the // matrix-multiplication operator. s.Take(1, syntax.ClassOperator) case r == '\\': // A backslash at the end of a line joins it to the next one. It is // structure rather than computation, and colouring it says that the // line does not end where it looks like it ends. s.Take(1, syntax.ClassPunctuation) case syntax.IsOperatorRune(r): s.TakeWhile(syntax.ClassOperator, syntax.IsOperatorRune) case syntax.IsPunctuationRune(r): s.Take(1, syntax.ClassPunctuation) default: // A rune nothing here claims — a currency sign in a comment-free line, // an accented letter in an identifier — is stepped over uncoloured // rather than guessed at. s.Advance(1) } } // --- decorators ------------------------------------------------------------- // isDecoratorStart reports whether a decorator opens at the scanner's position. // // The same rune is Python's matrix-multiplication operator, and the two are // told apart by where they are: a decorator is the first thing on its line. func isDecoratorStart(s *syntax.LineScanner) bool { if s.Peek(0) != '@' || !atLineStart(s) { return false } next := s.Peek(1) return syntax.IsLetter(next) || next == '_' } // takeDecorator colours @property and the dotted name of @app.route. // // It stops at the opening parenthesis rather than swallowing to the end of the // line: the arguments of @pytest.mark.parametrize("n", [1, 2]) are ordinary // Python, and colouring them as part of the decorator would hide a string and a // list inside one flat run. func takeDecorator(s *syntax.LineScanner) { start := s.Pos() s.Advance(1) // the @ for !s.AtEnd() && (syntax.IsWordRune(s.Peek(0)) || s.Peek(0) == '.') { s.Advance(1) } s.Emit(start, s.Pos(), syntax.ClassAttribute) } // --- where we are on the line ----------------------------------------------- // atLineStart reports whether nothing but indentation comes before the // scanner's position. // // Two decisions need it: a decorator is the first thing on its line, and so is // the soft keyword that opens a match statement. func atLineStart(s *syntax.LineScanner) bool { for at := -1; s.Pos()+at >= 0; at-- { if r := s.Peek(at); r != ' ' && r != '\t' { return false } } return true } // lineEndsWithColon reports whether the last thing on the line is the colon // that opens a block. // // It reads backwards from the end of the line, which is what makes it cheap // enough to ask about every word — and also what gives it its one boundary: a // trailing comment hides the colon from it, so `match x: # dispatch` colours // match as an identifier. That is the safe direction to be wrong in, and it is // documented rather than fixed, because telling a real trailing comment from a // # inside a string means scanning the line forwards, which is the work this // question is meant to avoid. func lineEndsWithColon(s *syntax.LineScanner) bool { for at := s.Len() - s.Pos() - 1; at >= -s.Pos(); at-- { switch r := s.Peek(at); r { case ' ', '\t': case ':': return true default: return false } } return false }