package pythonlang // Numbers and words: what a run of letters or digits turns out to be. import ( "strings" "rickub.com/turbo-editors/turbo-core/syntax" ) // --- numbers ---------------------------------------------------------------- // takeNumber colours a numeric literal, base prefix, underscores, exponent and // imaginary suffix included: 1_000, 0xFF, 0b1010, .5, 1.5e-3, 3j. func takeNumber(s *syntax.LineScanner) { start := s.Pos() // A float may have exactly one dot, and 1. is as valid as 1.0 — so the dot // is counted rather than required to have a digit after it. A second one is // where the number stops, which is what keeps `1..2` from being one token. seenDot := s.Peek(0) == '.' // 0xE-1 is a hexadecimal literal minus one, not an exponent: the sign rule // below has to know that the E it just saw was a digit. hex := s.Peek(0) == '0' && (s.Peek(1) == 'x' || s.Peek(1) == 'X') s.Advance(1) for !s.AtEnd() { r := s.Peek(0) switch { case syntax.IsWordRune(r): s.Advance(1) case r == '.' && !seenDot: seenDot = true s.Advance(1) case (r == '+' || r == '-') && !hex && isExponent(s.Peek(-1)): s.Advance(1) default: s.Emit(start, s.Pos(), syntax.ClassNumber) return } } s.Emit(start, s.Pos(), syntax.ClassNumber) } // isExponent reports whether a rune is the e of an exponent, which is what // makes the sign after it part of the number rather than an operator. func isExponent(r rune) bool { return r == 'e' || r == 'E' } // --- words ------------------------------------------------------------------ // takeWord colours an identifier, deciding what kind of thing it is from the // word itself, from the rune after it, and — for the two soft keywords — from // where it sits on the line. func takeWord(s *syntax.LineScanner) { start := s.Pos() // Asked before the word is consumed, because afterwards the scanner is no // longer at its first rune. first := atLineStart(s) for !s.AtEnd() && syntax.IsWordRune(s.Peek(0)) { s.Advance(1) } word := wordAt(s, start) class := classOfWord(word, s.Peek(0)) if isSoftKeyword(word) && first && lineEndsWithColon(s) { class = syntax.ClassKeyword } s.Emit(start, s.Pos(), class) } // wordAt returns the word running from start to the scanner's position. func wordAt(s *syntax.LineScanner, start int) string { var b strings.Builder for at := start; at < s.Pos(); at++ { b.WriteRune(s.Peek(at - s.Pos())) } return b.String() } // isSoftKeyword reports whether a word is one of the two Python added without // reserving. // // `match` and `case` open a match statement, and are ordinary names everywhere // else — `match = re.match(pattern, text)` is the line that made this a rule // rather than a table entry. What tells them apart is the shape of a // statement: it starts the line, and the line ends with the colon that opens // its block. Both conditions are checked, and `type` is deliberately not here: // it is a builtin as well as a soft keyword, and reading as a builtin is right // in both of its jobs. func isSoftKeyword(word string) bool { return word == "match" || word == "case" } // classOfWord decides what a word is, given the rune that follows it. // // The order is the design. A word the language names is what the language says // it is, whatever follows it. Then the naming conventions, which in Python are // strong enough to answer a question that the syntax cannot: a class is called // exactly the way a function is, so `ValueError("nope")` and `parse("nope")` // are the same shape, and only CapWords tells them apart. Only after that does // a parenthesis make a name a function. // // This is the one place Turbo Python and Turbo Rust order the same three rules // differently, and the reason is Rust's `Some(x)`: there, a parenthesis after a // capitalised name is usually a constructor the language names, so it is // answered from the table before the convention is consulted. func classOfWord(word string, next rune) syntax.Class { if class, known := knownWords[word]; known { return class } for _, convention := range conventions { if convention.spelt(word) { return convention.class } } if next == '(' { return syntax.ClassFunction } return syntax.ClassIdentifier } // conventions are the ways Python spells what a name is, consulted in order for // a word the language does not name itself. // // A list rather than a chain of ifs because the order *is* the rule, and a list // is where a reader looks for one: a dunder is the language's whatever else it // looks like, and a name in capitals is a constant before it is a type. var conventions = []struct { spelt func(string) bool class syntax.Class }{ {isDunder, syntax.ClassBuiltin}, {isScreamingCase, syntax.ClassConstant}, {startsUpperCase, syntax.ClassType}, } // isDunder reports whether a word is one of the names the language reserves to // itself by spelling: __init__, __name__, __repr__. // // They are the language's own hooks rather than anybody's identifiers, and a // reader looking for where a class begins finds __init__ faster when it is not // the same colour as the method below it. func isDunder(word string) bool { return len(word) > 4 && strings.HasPrefix(word, "__") && strings.HasSuffix(word, "__") } // isScreamingCase reports whether a word is written the way PEP 8 writes a // constant: MAX_SIZE, HTTP_PORT, PI. // // Turbo Rust has no rule like this one and documents SCREAMING_SNAKE_CASE as a // known wrong answer — it colours such a word as a type. Python's convention is // separated from its class convention by more than Rust's is, so the wrong // answer is worth removing rather than inheriting. What it costs is a class // named in capitals, which is rare enough to document. func isScreamingCase(word string) bool { if len(word) < 2 { return false } letters := 0 for _, r := range word { switch { case r >= 'A' && r <= 'Z': letters++ case r == '_' || r >= '0' && r <= '9': default: return false } } return letters > 0 } // startsUpperCase reports whether a word begins with an ASCII capital. func startsUpperCase(word string) bool { return word != "" && word[0] >= 'A' && word[0] <= 'Z' } // knownWords is every word the language itself names, and what each one is. // // It is one table rather than four because it answers one question. The four // groups below are kept apart only so that each can carry the reasoning that // belongs to it. // // The exception hierarchy is deliberately absent. ValueError, KeyError and the // seventy others are CapWords, so the convention rule in classOfWord already // colours them as types — and a table naming them would go out of date the next // time Python adds one. var knownWords = merge( classify(syntax.ClassKeyword, keywords), classify(syntax.ClassConstant, constants), classify(syntax.ClassType, builtinTypes), classify(syntax.ClassBuiltin, builtinFunctions), ) // keywords are Python's reserved words — the ones that cannot be used as a // name. and, or, not, in and is are here rather than among the operators // because that is what the language calls them, and because a theme that quiets // keywords should quiet them. // // match and case are not here: they are reserved in no context at all, and are // decided by isSoftKeyword. var keywords = words( "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", ) // constants are the values the language names, plus the flag it sets for you. var constants = words("True", "False", "None", "NotImplemented", "Ellipsis", "__debug__") // builtinTypes are the types you can call without importing anything. // // self and cls are here as *builtins* rather than as types, in builtinFunctions // below — they name a value, not a type. var builtinTypes = words( "bool", "bytearray", "bytes", "complex", "dict", "float", "frozenset", "int", "list", "memoryview", "object", "range", "set", "slice", "str", "tuple", "type", ) // builtinFunctions are the names in the builtins module, plus the two argument // names every Python reader reads as the language's own. // // self and cls are a convention rather than a rule — a method may name its // first parameter anything — but the convention is universal enough that every // other highlighter colours them, and a reader who meets `self` reads it the // way a Rust reader reads `Some`. That parallel is the argument; the caveat is // that a parameter honestly named self in a plain function is coloured too. var builtinFunctions = words( "abs", "aiter", "anext", "all", "any", "ascii", "bin", "breakpoint", "callable", "chr", "classmethod", "compile", "delattr", "dir", "divmod", "enumerate", "eval", "exec", "filter", "format", "getattr", "globals", "hasattr", "hash", "help", "hex", "id", "input", "isinstance", "issubclass", "iter", "len", "locals", "map", "max", "min", "next", "oct", "open", "ord", "pow", "print", "property", "repr", "reversed", "round", "setattr", "sorted", "staticmethod", "sum", "super", "vars", "zip", "self", "cls", ) // words gathers a group of them, which reads better at the call sites above // than a slice literal does. func words(list ...string) []string { return list } // classify pairs every word in a group with the class it belongs to. func classify(class syntax.Class, list []string) map[string]syntax.Class { out := make(map[string]syntax.Class, len(list)) for _, word := range list { out[word] = class } return out } // merge folds the groups into one table. An earlier group wins a word a later // one repeats, which is what keeps a keyword a keyword. func merge(groups ...map[string]syntax.Class) map[string]syntax.Class { out := map[string]syntax.Class{} for _, group := range groups { for word, class := range group { if _, taken := out[word]; !taken { out[word] = class } } } return out }