| 📦 Turbo MoonBit cc1f595 k33g 10h ago | 1 | package moonbitlang |
| 2 | |
| 3 | // Numbers and words: what a run of digits or letters turns out to be. |
| 4 | |
| 5 | import ( |
| 6 | "strings" |
| 7 | |
| 8 | "rickub.com/turbo-editors/turbo-core/syntax" |
| 9 | ) |
| 10 | |
| 11 | // --- numbers ---------------------------------------------------------------- |
| 12 | |
| 13 | // numberSuffixes are the literal suffixes MoonBit recognises, longest first so |
| 14 | // that UL is matched before U. |
| 15 | // |
| 16 | // They are upper case and nothing else: the grammar spells out "Uppercase |
| 17 | // suffixes select UInt (U), Int64 (L), UInt64 (UL), BigInt (N), or Float (F)", |
| 18 | // so 123u is the number 123 followed by the identifier u, and colouring it |
| 19 | // otherwise would be inventing a literal the compiler will reject. |
| 20 | var numberSuffixes = []string{"UL", "U", "L", "N", "F"} |
| 21 | |
| 22 | // takeNumber colours a numeric literal: 1_000, 0xFF, 0o17, 0b1010, 1.5e-3, |
| 23 | // 0x1.8p3F, 42UL. |
| 24 | // |
| 25 | // It follows the grammar rather than being generous, because one case needs it |
| 26 | // to. "Before .., an integer ends first, so 1..=2 begins with 1 and ..=" — a |
| 27 | // scanner that swallowed any dot would read 1. as a double and leave .=2 |
| 28 | // behind, and a range would be miscoloured everywhere it appeared. |
| 29 | // |
| 30 | // The whole literal is one span, so every step here advances the scanner |
| 31 | // without colouring and the single Emit at the end covers what they consumed. |
| 32 | func takeNumber(s *syntax.LineScanner) { |
| 33 | start := s.Pos() |
| 34 | digit, hexadecimal := takeIntegerPart(s) |
| 35 | |
| 36 | // A floating-point literal always has a point, and the point is only part |
| 37 | // of the number when a second one does not follow it. |
| 38 | if s.Peek(0) == '.' && s.Peek(1) != '.' { |
| 39 | s.Advance(1) |
| 40 | advanceWhile(s, digit) |
| 41 | } |
| 42 | |
| 43 | takeExponent(s, hexadecimal) |
| 44 | takeNumberSuffix(s) |
| 45 | s.Emit(start, s.Pos(), syntax.ClassNumber) |
| 46 | } |
| 47 | |
| 48 | // takeIntegerPart consumes the digits before any point. It returns the |
| 49 | // predicate saying which runes count as digits for the rest of the literal, and |
| 50 | // whether the literal is hexadecimal. |
| 51 | // |
| 52 | // The base prefix decides both. After 0x every hexadecimal digit is a digit, |
| 53 | // which is what makes the F of 0x1.F part of the number rather than a Float |
| 54 | // suffix — and an exponent is introduced by p rather than by e, because e is |
| 55 | // itself a hexadecimal digit. |
| 56 | func takeIntegerPart(s *syntax.LineScanner) (digit func(rune) bool, hexadecimal bool) { |
| 57 | if s.Peek(0) == '0' { |
| 58 | switch s.Peek(1) { |
| 59 | case 'x', 'X': |
| 60 | return takeBase(s, isHexDigit), true |
| 61 | case 'o', 'O': |
| 62 | return takeBase(s, isOctalDigit), false |
| 63 | case 'b', 'B': |
| 64 | return takeBase(s, isBinaryDigit), false |
| 65 | } |
| 66 | } |
| 67 | |
| 68 | advanceWhile(s, isDecimalDigit) |
| 69 | return isDecimalDigit, false |
| 70 | } |
| 71 | |
| 72 | // takeBase consumes a base prefix and the digits after it, and hands back the |
| 73 | // predicate that recognised them. |
| 74 | func takeBase(s *syntax.LineScanner, digit func(rune) bool) func(rune) bool { |
| 75 | s.Advance(2) // the 0 and its base letter |
| 76 | advanceWhile(s, digit) |
| 77 | return digit |
| 78 | } |
| 79 | |
| 80 | // takeExponent consumes an exponent when one is there: e or E for a decimal |
| 81 | // literal, p or P for a hexadecimal one, each with an optional sign. |
| 82 | // |
| 83 | // The sign is only taken when a digit follows it, so 1e-x stops at the e and |
| 84 | // leaves the - and the x to be coloured as an operator and a name. |
| 85 | func takeExponent(s *syntax.LineScanner, hexadecimal bool) { |
| 86 | if !isExponentLetter(s.Peek(0), hexadecimal) { |
| 87 | return |
| 88 | } |
| 89 | |
| 90 | offset := 1 |
| 91 | if s.Peek(offset) == '+' || s.Peek(offset) == '-' { |
| 92 | offset++ |
| 93 | } |
| 94 | if !syntax.IsDigit(s.Peek(offset)) { |
| 95 | return |
| 96 | } |
| 97 | |
| 98 | s.Advance(offset) |
| 99 | advanceWhile(s, isDecimalDigit) |
| 100 | } |
| 101 | |
| 102 | // takeNumberSuffix consumes UL, U, L, N or F when the literal ends in one. |
| 103 | // |
| 104 | // A suffix must not be followed by another word rune: 1Length is not the Int64 |
| 105 | // 1 followed by ength, it is a number and then a name the compiler will |
| 106 | // complain about, and stopping short of it is the reading that says so. |
| 107 | func takeNumberSuffix(s *syntax.LineScanner) { |
| 108 | for _, suffix := range numberSuffixes { |
| 109 | if s.HasPrefix(0, suffix) && !syntax.IsWordRune(s.Peek(len(suffix))) { |
| 110 | s.Advance(len(suffix)) |
| 111 | return |
| 112 | } |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | // advanceWhile steps over runes that match, without colouring any of them. It |
| 117 | // is what a construct emitted as a single span uses in place of TakeWhile, |
| 118 | // which would colour each run it consumed and leave the Emit overlapping it. |
| 119 | func advanceWhile(s *syntax.LineScanner, matches func(rune) bool) { |
| 120 | for !s.AtEnd() && matches(s.Peek(0)) { |
| 121 | s.Advance(1) |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | // isExponentLetter reports whether a rune introduces an exponent, which depends |
| 126 | // on the base: a hexadecimal literal uses p, because e is one of its digits. |
| 127 | func isExponentLetter(r rune, hexadecimal bool) bool { |
| 128 | if hexadecimal { |
| 129 | return r == 'p' || r == 'P' |
| 130 | } |
| 131 | return r == 'e' || r == 'E' |
| 132 | } |
| 133 | |
| 134 | // isDecimalDigit reports whether a rune may appear in a decimal literal after |
| 135 | // its first digit. An underscore may, and "underscores may repeat or trail". |
| 136 | func isDecimalDigit(r rune) bool { return syntax.IsDigit(r) || r == '_' } |
| 137 | |
| 138 | // isHexDigit reports whether a rune may appear in a hexadecimal literal. |
| 139 | func isHexDigit(r rune) bool { |
| 140 | return isDecimalDigit(r) || |
| 141 | r >= 'a' && r <= 'f' || |
| 142 | r >= 'A' && r <= 'F' |
| 143 | } |
| 144 | |
| 145 | // isOctalDigit reports whether a rune may appear in an octal literal. |
| 146 | func isOctalDigit(r rune) bool { return r >= '0' && r <= '7' || r == '_' } |
| 147 | |
| 148 | // isBinaryDigit reports whether a rune may appear in a binary literal. |
| 149 | func isBinaryDigit(r rune) bool { return r == '0' || r == '1' || r == '_' } |
| 150 | |
| 151 | // --- words ------------------------------------------------------------------ |
| 152 | |
| 153 | // bangKeywords are the two keywords that end in an exclamation mark. |
| 154 | // |
| 155 | // The grammar lists try! and guard! among the keywords, and ! is an operator |
| 156 | // rune — so without this the mark would be coloured as an operator hanging off |
| 157 | // the end of a keyword, which is not what the language sees there. |
| 158 | var bangKeywords = map[string]bool{"try": true, "guard": true} |
| 159 | |
| 160 | // takeWord colours an identifier, deciding what kind of thing it is from the |
| 161 | // word itself and from the rune that follows it. |
| 162 | func takeWord(s *syntax.LineScanner) { |
| 163 | start := s.Pos() |
| 164 | advanceWhile(s, syntax.IsWordRune) |
| 165 | word := wordAt(s, start) |
| 166 | |
| 167 | if bangKeywords[word] && s.Peek(0) == '!' { |
| 168 | s.Advance(1) |
| 169 | s.Emit(start, s.Pos(), syntax.ClassKeyword) |
| 170 | return |
| 171 | } |
| 172 | if isLabel(s, word) { |
| 173 | s.Advance(1) // the ~ |
| 174 | s.Emit(start, s.Pos(), syntax.ClassAttribute) |
| 175 | return |
| 176 | } |
| 177 | |
| 178 | s.Emit(start, s.Pos(), classOfWord(word, s.Peek(0))) |
| 179 | } |
| 180 | |
| 181 | // takeMember colours the name after a dot: a field, or a method. |
| 182 | // |
| 183 | // It is takeWord without the keyword table, because MoonBit's dot-identifiers |
| 184 | // "use the identifier case rules without consulting the keyword table, so .if |
| 185 | // is valid". A record with a field called `type` is ordinary MoonBit, and |
| 186 | // colouring that field as a keyword would be a claim about the language that |
| 187 | // the language contradicts. |
| 188 | func takeMember(s *syntax.LineScanner) { |
| 189 | start := s.Pos() |
| 190 | advanceWhile(s, syntax.IsWordRune) |
| 191 | s.Emit(start, s.Pos(), classOfMember(wordAt(s, start), s.Peek(0))) |
| 192 | } |
| 193 | |
| 194 | // wordAt returns the word running from start to the scanner's position. |
| 195 | func wordAt(s *syntax.LineScanner, start int) string { |
| 196 | var b strings.Builder |
| 197 | for at := start; at < s.Pos(); at++ { |
| 198 | b.WriteRune(s.Peek(at - s.Pos())) |
| 199 | } |
| 200 | return b.String() |
| 201 | } |
| 202 | |
| 203 | // isLabel reports whether the word just consumed is a labelled argument's name, |
| 204 | // which is to say whether a tilde touches it. |
| 205 | // |
| 206 | // The tilde is MoonBit's alone: it appears in no operator the language has, so |
| 207 | // a tilde against the end of a name can only be a label. The two exclusions are |
| 208 | // the grammar's own — "ASCII-uppercase identifiers and keywords cannot form |
| 209 | // labels" — and they matter, because without the first, Foo~ in a piece of |
| 210 | // half-typed code would colour a type as a label. |
| 211 | func isLabel(s *syntax.LineScanner, word string) bool { |
| 212 | if s.Peek(0) != '~' || word == "" { |
| 213 | return false |
| 214 | } |
| 215 | if startsUpperCase(word) { |
| 216 | return false |
| 217 | } |
| 218 | _, reserved := knownWords[word] |
| 219 | return !reserved |
| 220 | } |
| 221 | |
| 222 | // classOfWord decides what a word is, given the rune that follows it. |
| 223 | // |
| 224 | // The order is the design, and MoonBit lets it be shorter than any other |
| 225 | // scanner in this family. A word the language names is what the language says |
| 226 | // it is. After that comes the case rule — and in MoonBit that is a *lexical* |
| 227 | // rule rather than a convention: a uident "begins with an ASCII uppercase |
| 228 | // letter", and only a type, a trait or an enum constructor may be spelt that |
| 229 | // way. So there is no table of built-in types here, and there does not need to |
| 230 | // be: Int, StringBuilder and a type somebody wrote this morning are all |
| 231 | // capitalised, and all coloured by the same line. |
| 232 | // |
| 233 | // What that costs is that a constructor of your own enum is coloured as a type. |
| 234 | // Nothing in the syntax separates Circle(1.0) from a type applied to arguments, |
| 235 | // and inventing a separation would mean being wrong in both directions instead |
| 236 | // of one. |
| 237 | func classOfWord(word string, next rune) syntax.Class { |
| 238 | if class, known := knownWords[word]; known { |
| 239 | return class |
| 240 | } |
| 241 | if startsUpperCase(word) { |
| 242 | return syntax.ClassType |
| 243 | } |
| 244 | if next == '(' { |
| 245 | return syntax.ClassFunction |
| 246 | } |
| 247 | return syntax.ClassIdentifier |
| 248 | } |
| 249 | |
| 250 | // classOfMember decides what the name after a dot is. It is classOfWord with |
| 251 | // the keyword table left out; see takeMember. |
| 252 | func classOfMember(word string, next rune) syntax.Class { |
| 253 | if startsUpperCase(word) { |
| 254 | return syntax.ClassType |
| 255 | } |
| 256 | if next == '(' { |
| 257 | return syntax.ClassFunction |
| 258 | } |
| 259 | return syntax.ClassIdentifier |
| 260 | } |
| 261 | |
| 262 | // startsUpperCase reports whether a word begins with an ASCII capital, which is |
| 263 | // what makes it a uident. |
| 264 | func startsUpperCase(word string) bool { |
| 265 | return word != "" && word[0] >= 'A' && word[0] <= 'Z' |
| 266 | } |
| 267 | |
| 268 | // knownWords is every word the language itself names, and what each one is. |
| 269 | // |
| 270 | // It is one table rather than three because it answers one question. The three |
| 271 | // groups below are kept apart only so that each can carry the reasoning that |
| 272 | // belongs to it. |
| 273 | var knownWords = merge( |
| 274 | classify(syntax.ClassKeyword, keywords), |
| 275 | classify(syntax.ClassConstant, constants), |
| 276 | classify(syntax.ClassBuiltin, builtinValues), |
| 277 | ) |
| 278 | |
| 279 | // keywords are the words MoonBit reserves, taken from the grammar's keyword |
| 280 | // production. |
| 281 | // |
| 282 | // true and false are in it there and are not here: they are keywords to the |
| 283 | // lexer and values to the reader, and every editor in this family colours them |
| 284 | // as constants. try! and guard! are not here either — the mark is glued on by |
| 285 | // takeWord, because a table cannot hold a word whose last rune is an operator. |
| 286 | // |
| 287 | // package is here although in a .mbt file it is only a *reserved* word, which |
| 288 | // the lexer treats as an identifier and warns about. It is a real keyword in |
| 289 | // the .mbti interface files this editor also colours, and in a .mbt file |
| 290 | // colouring it says exactly what the compiler is about to: this word is not |
| 291 | // yours to use. |
| 292 | // |
| 293 | // The rest of the reserved list — move, ref, static, unsafe, await, and the |
| 294 | // forty others — is deliberately absent. Those are identifiers that earn a |
| 295 | // warning, and a scanner that coloured them as keywords would be telling a |
| 296 | // reader they cannot write `let ref = 1` when they can. |
| 297 | var keywords = words( |
| 298 | "and", "as", "async", "break", "catch", "const", "continue", "declare", |
| 299 | "defer", "derive", "else", "enum", "enumview", "extend", "extenum", |
| 300 | "extern", "fn", "for", "guard", "if", "impl", "import", "in", "is", "let", |
| 301 | "letrec", "lexscan", "loop", "match", "mut", "nobreak", "nocancel", |
| 302 | "noraise", "package", "priv", "proof_assert", "proof_let", "pub", "raise", |
| 303 | "readonly", "return", "struct", "suberror", "test", "throw", "trait", |
| 304 | "try", "type", "using", "where", "while", "with", |
| 305 | ) |
| 306 | |
| 307 | // constants are the values a reader meets as the language's own. |
| 308 | // |
| 309 | // None, Some, Ok and Err belong to Option and Result rather than to the |
| 310 | // language, but a reader meets them everywhere and reads them as built in, the |
| 311 | // same argument Turbo Rust records for the same four names. |
| 312 | var constants = words("true", "false", "None", "Some", "Ok", "Err") |
| 313 | |
| 314 | // builtinValues are the lower-case names the prelude puts in scope without an |
| 315 | // import, read out of moonbitlang/core/prelude rather than remembered. |
| 316 | // |
| 317 | // The prelude's deprecated names — dump, not, tap, then, to_repr — are left |
| 318 | // out on purpose: colouring them as builtins would present as the language's |
| 319 | // own four things it is trying to retire. There is no print here for the same |
| 320 | // kind of reason, and it is the one worth stating: MoonBit has println and has |
| 321 | // never had print, so a table written from habit would have coloured a name |
| 322 | // that does not exist. |
| 323 | var builtinValues = words( |
| 324 | "abort", "assert_eq", "assert_false", "assert_not_eq", "assert_true", |
| 325 | "compare", "debug", "debug_assert", "debug_inspect", "fail", "hash", |
| 326 | "ignore", "inspect", "json_inspect", "null", "panic", "physical_equal", |
| 327 | "println", "repr", |
| 328 | ) |
| 329 | |
| 330 | // words gathers a group of them, which reads better at the call sites above |
| 331 | // than a slice literal does. |
| 332 | func words(list ...string) []string { return list } |
| 333 | |
| 334 | // classify pairs every word in a group with the class it belongs to. |
| 335 | func classify(class syntax.Class, list []string) map[string]syntax.Class { |
| 336 | out := make(map[string]syntax.Class, len(list)) |
| 337 | for _, word := range list { |
| 338 | out[word] = class |
| 339 | } |
| 340 | return out |
| 341 | } |
| 342 | |
| 343 | // merge folds the groups into one table. An earlier group wins a word a later |
| 344 | // one repeats, which is what keeps a keyword a keyword. |
| 345 | func merge(groups ...map[string]syntax.Class) map[string]syntax.Class { |
| 346 | out := map[string]syntax.Class{} |
| 347 | for _, group := range groups { |
| 348 | for word, class := range group { |
| 349 | if _, taken := out[word]; !taken { |
| 350 | out[word] = class |
| 351 | } |
| 352 | } |
| 353 | } |
| 354 | return out |
| 355 | } |