| 📦 Turbo Python 6fc62ea k33g 8h ago | 1 | package pythonlang |
| 2 | |
| 3 | // Numbers and words: what a run of letters or digits turns out to be. |
| 4 | |
| 5 | import ( |
| 6 | "strings" |
| 7 | |
| 8 | "rickub.com/turbo-editors/turbo-core/syntax" |
| 9 | ) |
| 10 | |
| 11 | // --- numbers ---------------------------------------------------------------- |
| 12 | |
| 13 | // takeNumber colours a numeric literal, base prefix, underscores, exponent and |
| 14 | // imaginary suffix included: 1_000, 0xFF, 0b1010, .5, 1.5e-3, 3j. |
| 15 | func takeNumber(s *syntax.LineScanner) { |
| 16 | start := s.Pos() |
| 17 | // A float may have exactly one dot, and 1. is as valid as 1.0 — so the dot |
| 18 | // is counted rather than required to have a digit after it. A second one is |
| 19 | // where the number stops, which is what keeps `1..2` from being one token. |
| 20 | seenDot := s.Peek(0) == '.' |
| 21 | // 0xE-1 is a hexadecimal literal minus one, not an exponent: the sign rule |
| 22 | // below has to know that the E it just saw was a digit. |
| 23 | hex := s.Peek(0) == '0' && (s.Peek(1) == 'x' || s.Peek(1) == 'X') |
| 24 | s.Advance(1) |
| 25 | |
| 26 | for !s.AtEnd() { |
| 27 | r := s.Peek(0) |
| 28 | switch { |
| 29 | case syntax.IsWordRune(r): |
| 30 | s.Advance(1) |
| 31 | case r == '.' && !seenDot: |
| 32 | seenDot = true |
| 33 | s.Advance(1) |
| 34 | case (r == '+' || r == '-') && !hex && isExponent(s.Peek(-1)): |
| 35 | s.Advance(1) |
| 36 | default: |
| 37 | s.Emit(start, s.Pos(), syntax.ClassNumber) |
| 38 | return |
| 39 | } |
| 40 | } |
| 41 | s.Emit(start, s.Pos(), syntax.ClassNumber) |
| 42 | } |
| 43 | |
| 44 | // isExponent reports whether a rune is the e of an exponent, which is what |
| 45 | // makes the sign after it part of the number rather than an operator. |
| 46 | func isExponent(r rune) bool { return r == 'e' || r == 'E' } |
| 47 | |
| 48 | // --- words ------------------------------------------------------------------ |
| 49 | |
| 50 | // takeWord colours an identifier, deciding what kind of thing it is from the |
| 51 | // word itself, from the rune after it, and — for the two soft keywords — from |
| 52 | // where it sits on the line. |
| 53 | func takeWord(s *syntax.LineScanner) { |
| 54 | start := s.Pos() |
| 55 | // Asked before the word is consumed, because afterwards the scanner is no |
| 56 | // longer at its first rune. |
| 57 | first := atLineStart(s) |
| 58 | |
| 59 | for !s.AtEnd() && syntax.IsWordRune(s.Peek(0)) { |
| 60 | s.Advance(1) |
| 61 | } |
| 62 | word := wordAt(s, start) |
| 63 | |
| 64 | class := classOfWord(word, s.Peek(0)) |
| 65 | if isSoftKeyword(word) && first && lineEndsWithColon(s) { |
| 66 | class = syntax.ClassKeyword |
| 67 | } |
| 68 | s.Emit(start, s.Pos(), class) |
| 69 | } |
| 70 | |
| 71 | // wordAt returns the word running from start to the scanner's position. |
| 72 | func wordAt(s *syntax.LineScanner, start int) string { |
| 73 | var b strings.Builder |
| 74 | for at := start; at < s.Pos(); at++ { |
| 75 | b.WriteRune(s.Peek(at - s.Pos())) |
| 76 | } |
| 77 | return b.String() |
| 78 | } |
| 79 | |
| 80 | // isSoftKeyword reports whether a word is one of the two Python added without |
| 81 | // reserving. |
| 82 | // |
| 83 | // `match` and `case` open a match statement, and are ordinary names everywhere |
| 84 | // else — `match = re.match(pattern, text)` is the line that made this a rule |
| 85 | // rather than a table entry. What tells them apart is the shape of a |
| 86 | // statement: it starts the line, and the line ends with the colon that opens |
| 87 | // its block. Both conditions are checked, and `type` is deliberately not here: |
| 88 | // it is a builtin as well as a soft keyword, and reading as a builtin is right |
| 89 | // in both of its jobs. |
| 90 | func isSoftKeyword(word string) bool { return word == "match" || word == "case" } |
| 91 | |
| 92 | // classOfWord decides what a word is, given the rune that follows it. |
| 93 | // |
| 94 | // The order is the design. A word the language names is what the language says |
| 95 | // it is, whatever follows it. Then the naming conventions, which in Python are |
| 96 | // strong enough to answer a question that the syntax cannot: a class is called |
| 97 | // exactly the way a function is, so `ValueError("nope")` and `parse("nope")` |
| 98 | // are the same shape, and only CapWords tells them apart. Only after that does |
| 99 | // a parenthesis make a name a function. |
| 100 | // |
| 101 | // This is the one place Turbo Python and Turbo Rust order the same three rules |
| 102 | // differently, and the reason is Rust's `Some(x)`: there, a parenthesis after a |
| 103 | // capitalised name is usually a constructor the language names, so it is |
| 104 | // answered from the table before the convention is consulted. |
| 105 | func classOfWord(word string, next rune) syntax.Class { |
| 106 | if class, known := knownWords[word]; known { |
| 107 | return class |
| 108 | } |
| 109 | for _, convention := range conventions { |
| 110 | if convention.spelt(word) { |
| 111 | return convention.class |
| 112 | } |
| 113 | } |
| 114 | if next == '(' { |
| 115 | return syntax.ClassFunction |
| 116 | } |
| 117 | return syntax.ClassIdentifier |
| 118 | } |
| 119 | |
| 120 | // conventions are the ways Python spells what a name is, consulted in order for |
| 121 | // a word the language does not name itself. |
| 122 | // |
| 123 | // A list rather than a chain of ifs because the order *is* the rule, and a list |
| 124 | // is where a reader looks for one: a dunder is the language's whatever else it |
| 125 | // looks like, and a name in capitals is a constant before it is a type. |
| 126 | var conventions = []struct { |
| 127 | spelt func(string) bool |
| 128 | class syntax.Class |
| 129 | }{ |
| 130 | {isDunder, syntax.ClassBuiltin}, |
| 131 | {isScreamingCase, syntax.ClassConstant}, |
| 132 | {startsUpperCase, syntax.ClassType}, |
| 133 | } |
| 134 | |
| 135 | // isDunder reports whether a word is one of the names the language reserves to |
| 136 | // itself by spelling: __init__, __name__, __repr__. |
| 137 | // |
| 138 | // They are the language's own hooks rather than anybody's identifiers, and a |
| 139 | // reader looking for where a class begins finds __init__ faster when it is not |
| 140 | // the same colour as the method below it. |
| 141 | func isDunder(word string) bool { |
| 142 | return len(word) > 4 && strings.HasPrefix(word, "__") && strings.HasSuffix(word, "__") |
| 143 | } |
| 144 | |
| 145 | // isScreamingCase reports whether a word is written the way PEP 8 writes a |
| 146 | // constant: MAX_SIZE, HTTP_PORT, PI. |
| 147 | // |
| 148 | // Turbo Rust has no rule like this one and documents SCREAMING_SNAKE_CASE as a |
| 149 | // known wrong answer — it colours such a word as a type. Python's convention is |
| 150 | // separated from its class convention by more than Rust's is, so the wrong |
| 151 | // answer is worth removing rather than inheriting. What it costs is a class |
| 152 | // named in capitals, which is rare enough to document. |
| 153 | func isScreamingCase(word string) bool { |
| 154 | if len(word) < 2 { |
| 155 | return false |
| 156 | } |
| 157 | letters := 0 |
| 158 | for _, r := range word { |
| 159 | switch { |
| 160 | case r >= 'A' && r <= 'Z': |
| 161 | letters++ |
| 162 | case r == '_' || r >= '0' && r <= '9': |
| 163 | default: |
| 164 | return false |
| 165 | } |
| 166 | } |
| 167 | return letters > 0 |
| 168 | } |
| 169 | |
| 170 | // startsUpperCase reports whether a word begins with an ASCII capital. |
| 171 | func startsUpperCase(word string) bool { |
| 172 | return word != "" && word[0] >= 'A' && word[0] <= 'Z' |
| 173 | } |
| 174 | |
| 175 | // knownWords is every word the language itself names, and what each one is. |
| 176 | // |
| 177 | // It is one table rather than four because it answers one question. The four |
| 178 | // groups below are kept apart only so that each can carry the reasoning that |
| 179 | // belongs to it. |
| 180 | // |
| 181 | // The exception hierarchy is deliberately absent. ValueError, KeyError and the |
| 182 | // seventy others are CapWords, so the convention rule in classOfWord already |
| 183 | // colours them as types — and a table naming them would go out of date the next |
| 184 | // time Python adds one. |
| 185 | var knownWords = merge( |
| 186 | classify(syntax.ClassKeyword, keywords), |
| 187 | classify(syntax.ClassConstant, constants), |
| 188 | classify(syntax.ClassType, builtinTypes), |
| 189 | classify(syntax.ClassBuiltin, builtinFunctions), |
| 190 | ) |
| 191 | |
| 192 | // keywords are Python's reserved words — the ones that cannot be used as a |
| 193 | // name. and, or, not, in and is are here rather than among the operators |
| 194 | // because that is what the language calls them, and because a theme that quiets |
| 195 | // keywords should quiet them. |
| 196 | // |
| 197 | // match and case are not here: they are reserved in no context at all, and are |
| 198 | // decided by isSoftKeyword. |
| 199 | var keywords = words( |
| 200 | "and", "as", "assert", "async", "await", "break", "class", "continue", |
| 201 | "def", "del", "elif", "else", "except", "finally", "for", "from", "global", |
| 202 | "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", |
| 203 | "raise", "return", "try", "while", "with", "yield", |
| 204 | ) |
| 205 | |
| 206 | // constants are the values the language names, plus the flag it sets for you. |
| 207 | var constants = words("True", "False", "None", "NotImplemented", "Ellipsis", "__debug__") |
| 208 | |
| 209 | // builtinTypes are the types you can call without importing anything. |
| 210 | // |
| 211 | // self and cls are here as *builtins* rather than as types, in builtinFunctions |
| 212 | // below — they name a value, not a type. |
| 213 | var builtinTypes = words( |
| 214 | "bool", "bytearray", "bytes", "complex", "dict", "float", "frozenset", |
| 215 | "int", "list", "memoryview", "object", "range", "set", "slice", "str", |
| 216 | "tuple", "type", |
| 217 | ) |
| 218 | |
| 219 | // builtinFunctions are the names in the builtins module, plus the two argument |
| 220 | // names every Python reader reads as the language's own. |
| 221 | // |
| 222 | // self and cls are a convention rather than a rule — a method may name its |
| 223 | // first parameter anything — but the convention is universal enough that every |
| 224 | // other highlighter colours them, and a reader who meets `self` reads it the |
| 225 | // way a Rust reader reads `Some`. That parallel is the argument; the caveat is |
| 226 | // that a parameter honestly named self in a plain function is coloured too. |
| 227 | var builtinFunctions = words( |
| 228 | "abs", "aiter", "anext", "all", "any", "ascii", "bin", "breakpoint", |
| 229 | "callable", "chr", "classmethod", "compile", "delattr", "dir", "divmod", |
| 230 | "enumerate", "eval", "exec", "filter", "format", "getattr", "globals", |
| 231 | "hasattr", "hash", "help", "hex", "id", "input", "isinstance", "issubclass", |
| 232 | "iter", "len", "locals", "map", "max", "min", "next", "oct", "open", "ord", |
| 233 | "pow", "print", "property", "repr", "reversed", "round", "setattr", |
| 234 | "sorted", "staticmethod", "sum", "super", "vars", "zip", |
| 235 | "self", "cls", |
| 236 | ) |
| 237 | |
| 238 | // words gathers a group of them, which reads better at the call sites above |
| 239 | // than a slice literal does. |
| 240 | func words(list ...string) []string { return list } |
| 241 | |
| 242 | // classify pairs every word in a group with the class it belongs to. |
| 243 | func classify(class syntax.Class, list []string) map[string]syntax.Class { |
| 244 | out := make(map[string]syntax.Class, len(list)) |
| 245 | for _, word := range list { |
| 246 | out[word] = class |
| 247 | } |
| 248 | return out |
| 249 | } |
| 250 | |
| 251 | // merge folds the groups into one table. An earlier group wins a word a later |
| 252 | // one repeats, which is what keeps a keyword a keyword. |
| 253 | func merge(groups ...map[string]syntax.Class) map[string]syntax.Class { |
| 254 | out := map[string]syntax.Class{} |
| 255 | for _, group := range groups { |
| 256 | for word, class := range group { |
| 257 | if _, taken := out[word]; !taken { |
| 258 | out[word] = class |
| 259 | } |
| 260 | } |
| 261 | } |
| 262 | return out |
| 263 | } |