| 📦 Turbo Rust 713ea5c k33g 10h ago | 1 | package rustlang |
| 2 | |
| 3 | // Numbers and words: what a run of letters or digits turns out to be. |
| 4 | |
| 5 | import ( |
| 6 | "strings" |
| 7 | |
| 8 | "rickub.com/turbo-editors/turbo-core/syntax" |
| 9 | ) |
| 10 | |
| 11 | // --- numbers ---------------------------------------------------------------- |
| 12 | |
| 13 | // takeNumber colours a numeric literal, underscores, base prefix, exponent and |
| 14 | // type suffix included: 1_000, 0xFF_u8, 1.5e-3f64. |
| 15 | // |
| 16 | // The suffix is taken as part of the number rather than as an identifier |
| 17 | // beside it, because 3u8 is one literal and colouring the u8 as a type would |
| 18 | // split a thing that is not two things. |
| 19 | func takeNumber(s *syntax.LineScanner) { |
| 20 | start := s.Pos() |
| 21 | s.Advance(1) |
| 22 | |
| 23 | for !s.AtEnd() { |
| 24 | r := s.Peek(0) |
| 25 | switch { |
| 26 | case syntax.IsWordRune(r) || r == '.' && syntax.IsDigit(s.Peek(1)): |
| 27 | s.Advance(1) |
| 28 | case (r == '+' || r == '-') && isExponent(s.Peek(-1)): |
| 29 | s.Advance(1) |
| 30 | default: |
| 31 | s.Emit(start, s.Pos(), syntax.ClassNumber) |
| 32 | return |
| 33 | } |
| 34 | } |
| 35 | s.Emit(start, s.Pos(), syntax.ClassNumber) |
| 36 | } |
| 37 | |
| 38 | // isExponent reports whether a rune is the e of an exponent, which is what |
| 39 | // makes the sign after it part of the number rather than an operator. |
| 40 | func isExponent(r rune) bool { return r == 'e' || r == 'E' } |
| 41 | |
| 42 | // rangeWidth returns how many runes the range operator at the scanner's |
| 43 | // position takes: three for ..=, two otherwise. |
| 44 | func rangeWidth(s *syntax.LineScanner) int { |
| 45 | if s.Peek(2) == '=' { |
| 46 | return 3 |
| 47 | } |
| 48 | return 2 |
| 49 | } |
| 50 | |
| 51 | // --- words ------------------------------------------------------------------ |
| 52 | |
| 53 | // takeWord colours an identifier, deciding what kind of thing it is from the |
| 54 | // word itself and from the rune after it. |
| 55 | func takeWord(s *syntax.LineScanner) { |
| 56 | start := s.Pos() |
| 57 | for !s.AtEnd() && syntax.IsWordRune(s.Peek(0)) { |
| 58 | s.Advance(1) |
| 59 | } |
| 60 | word := wordAt(s, start) |
| 61 | end := s.Pos() |
| 62 | |
| 63 | // A macro takes the ! with it: println! is one name, and colouring the ! |
| 64 | // as an operator would make it read as a negation. |
| 65 | if s.Peek(0) == '!' && s.Peek(1) != '=' { |
| 66 | s.Advance(1) |
| 67 | s.Emit(start, s.Pos(), syntax.ClassBuiltin) |
| 68 | return |
| 69 | } |
| 70 | s.Emit(start, end, classOfWord(word, s.Peek(0))) |
| 71 | } |
| 72 | |
| 73 | // wordAt returns the word running from start to the scanner's position. |
| 74 | func wordAt(s *syntax.LineScanner, start int) string { |
| 75 | var b strings.Builder |
| 76 | for at := start; at < s.Pos(); at++ { |
| 77 | b.WriteRune(s.Peek(at - s.Pos())) |
| 78 | } |
| 79 | return b.String() |
| 80 | } |
| 81 | |
| 82 | // classOfWord decides what a word is, given the rune that follows it. |
| 83 | // |
| 84 | // The order is the design: a word the language names is what the language says |
| 85 | // it is, whatever follows it — which is what stops `u8::MAX` reading as a call |
| 86 | // — and only then does `(` make a name one. |
| 87 | func classOfWord(word string, next rune) syntax.Class { |
| 88 | if class, known := knownWords[word]; known { |
| 89 | return class |
| 90 | } |
| 91 | if next == '(' { |
| 92 | return syntax.ClassFunction |
| 93 | } |
| 94 | if startsUpperCase(word) { |
| 95 | // Rust's naming convention is strong enough to lean on: a type, a trait |
| 96 | // and an enum variant are all UpperCamelCase and nothing else is, so a |
| 97 | // leading capital says "type" more reliably here than any amount of |
| 98 | // looking at neighbouring tokens would. |
| 99 | return syntax.ClassType |
| 100 | } |
| 101 | return syntax.ClassIdentifier |
| 102 | } |
| 103 | |
| 104 | // startsUpperCase reports whether a word begins with an ASCII capital. |
| 105 | func startsUpperCase(word string) bool { |
| 106 | return word != "" && word[0] >= 'A' && word[0] <= 'Z' |
| 107 | } |
| 108 | |
| 109 | // knownWords is every word the language itself names, and what each one is. |
| 110 | // |
| 111 | // It is one table rather than three because it answers one question. The three |
| 112 | // groups below are kept apart only so that each can carry the reasoning that |
| 113 | // belongs to it. |
| 114 | var knownWords = merge( |
| 115 | classify(syntax.ClassKeyword, keywords), |
| 116 | classify(syntax.ClassConstant, constants), |
| 117 | classify(syntax.ClassType, primitiveTypes), |
| 118 | ) |
| 119 | |
| 120 | // keywords are Rust's reserved words, including the ones reserved for future |
| 121 | // use — a file using one will not compile, and colouring it as an identifier |
| 122 | // would be the friendlier of two wrong answers. |
| 123 | var keywords = words( |
| 124 | "as", "async", "await", "break", "const", "continue", "crate", "dyn", |
| 125 | "else", "enum", "extern", "fn", "for", "if", "impl", "in", "let", "loop", |
| 126 | "macro_rules", "match", "mod", "move", "mut", "pub", "ref", "return", |
| 127 | "static", "struct", "super", "trait", "type", "union", "unsafe", "use", |
| 128 | "where", "while", "yield", |
| 129 | // Reserved for future use. |
| 130 | "abstract", "become", "box", "do", "final", "override", "priv", "try", |
| 131 | "typeof", "unsized", "virtual", |
| 132 | ) |
| 133 | |
| 134 | // constants are the literals the language itself provides, plus the two enum |
| 135 | // variants everybody meets before they meet any other. |
| 136 | // |
| 137 | // None and Some are Option's, not the language's, but a reader looking at Rust |
| 138 | // reads them as they read true and false, and a theme that quiets constants |
| 139 | // should quiet them too. |
| 140 | var constants = words("true", "false", "None", "Some", "Ok", "Err") |
| 141 | |
| 142 | // primitiveTypes are the built-in types and the two self words. |
| 143 | // |
| 144 | // self and Self are keywords to the compiler; they are here because what a |
| 145 | // reader wants coloured is the *type* they stand for, and Self in an impl block |
| 146 | // reads as the type it names. |
| 147 | var primitiveTypes = words( |
| 148 | "bool", "char", "str", "f32", "f64", |
| 149 | "i8", "i16", "i32", "i64", "i128", "isize", |
| 150 | "u8", "u16", "u32", "u64", "u128", "usize", |
| 151 | "self", "Self", |
| 152 | ) |
| 153 | |
| 154 | // words gathers a group of them, which reads better at the call sites above |
| 155 | // than a slice literal does. |
| 156 | func words(list ...string) []string { return list } |
| 157 | |
| 158 | // classify pairs every word in a group with the class it belongs to. |
| 159 | func classify(class syntax.Class, list []string) map[string]syntax.Class { |
| 160 | out := make(map[string]syntax.Class, len(list)) |
| 161 | for _, word := range list { |
| 162 | out[word] = class |
| 163 | } |
| 164 | return out |
| 165 | } |
| 166 | |
| 167 | // merge folds the groups into one table. An earlier group wins a word a later |
| 168 | // one repeats, which is what keeps a keyword a keyword. |
| 169 | func merge(groups ...map[string]syntax.Class) map[string]syntax.Class { |
| 170 | out := map[string]syntax.Class{} |
| 171 | for _, group := range groups { |
| 172 | for word, class := range group { |
| 173 | if _, taken := out[word]; !taken { |
| 174 | out[word] = class |
| 175 | } |
| 176 | } |
| 177 | } |
| 178 | return out |
| 179 | } |