package moonbitlang // Numbers and words: what a run of digits or letters turns out to be. import ( "strings" "rickub.com/turbo-editors/turbo-core/syntax" ) // --- numbers ---------------------------------------------------------------- // numberSuffixes are the literal suffixes MoonBit recognises, longest first so // that UL is matched before U. // // They are upper case and nothing else: the grammar spells out "Uppercase // suffixes select UInt (U), Int64 (L), UInt64 (UL), BigInt (N), or Float (F)", // so 123u is the number 123 followed by the identifier u, and colouring it // otherwise would be inventing a literal the compiler will reject. var numberSuffixes = []string{"UL", "U", "L", "N", "F"} // takeNumber colours a numeric literal: 1_000, 0xFF, 0o17, 0b1010, 1.5e-3, // 0x1.8p3F, 42UL. // // It follows the grammar rather than being generous, because one case needs it // to. "Before .., an integer ends first, so 1..=2 begins with 1 and ..=" — a // scanner that swallowed any dot would read 1. as a double and leave .=2 // behind, and a range would be miscoloured everywhere it appeared. // // The whole literal is one span, so every step here advances the scanner // without colouring and the single Emit at the end covers what they consumed. func takeNumber(s *syntax.LineScanner) { start := s.Pos() digit, hexadecimal := takeIntegerPart(s) // A floating-point literal always has a point, and the point is only part // of the number when a second one does not follow it. if s.Peek(0) == '.' && s.Peek(1) != '.' { s.Advance(1) advanceWhile(s, digit) } takeExponent(s, hexadecimal) takeNumberSuffix(s) s.Emit(start, s.Pos(), syntax.ClassNumber) } // takeIntegerPart consumes the digits before any point. It returns the // predicate saying which runes count as digits for the rest of the literal, and // whether the literal is hexadecimal. // // The base prefix decides both. After 0x every hexadecimal digit is a digit, // which is what makes the F of 0x1.F part of the number rather than a Float // suffix — and an exponent is introduced by p rather than by e, because e is // itself a hexadecimal digit. func takeIntegerPart(s *syntax.LineScanner) (digit func(rune) bool, hexadecimal bool) { if s.Peek(0) == '0' { switch s.Peek(1) { case 'x', 'X': return takeBase(s, isHexDigit), true case 'o', 'O': return takeBase(s, isOctalDigit), false case 'b', 'B': return takeBase(s, isBinaryDigit), false } } advanceWhile(s, isDecimalDigit) return isDecimalDigit, false } // takeBase consumes a base prefix and the digits after it, and hands back the // predicate that recognised them. func takeBase(s *syntax.LineScanner, digit func(rune) bool) func(rune) bool { s.Advance(2) // the 0 and its base letter advanceWhile(s, digit) return digit } // takeExponent consumes an exponent when one is there: e or E for a decimal // literal, p or P for a hexadecimal one, each with an optional sign. // // The sign is only taken when a digit follows it, so 1e-x stops at the e and // leaves the - and the x to be coloured as an operator and a name. func takeExponent(s *syntax.LineScanner, hexadecimal bool) { if !isExponentLetter(s.Peek(0), hexadecimal) { return } offset := 1 if s.Peek(offset) == '+' || s.Peek(offset) == '-' { offset++ } if !syntax.IsDigit(s.Peek(offset)) { return } s.Advance(offset) advanceWhile(s, isDecimalDigit) } // takeNumberSuffix consumes UL, U, L, N or F when the literal ends in one. // // A suffix must not be followed by another word rune: 1Length is not the Int64 // 1 followed by ength, it is a number and then a name the compiler will // complain about, and stopping short of it is the reading that says so. func takeNumberSuffix(s *syntax.LineScanner) { for _, suffix := range numberSuffixes { if s.HasPrefix(0, suffix) && !syntax.IsWordRune(s.Peek(len(suffix))) { s.Advance(len(suffix)) return } } } // advanceWhile steps over runes that match, without colouring any of them. It // is what a construct emitted as a single span uses in place of TakeWhile, // which would colour each run it consumed and leave the Emit overlapping it. func advanceWhile(s *syntax.LineScanner, matches func(rune) bool) { for !s.AtEnd() && matches(s.Peek(0)) { s.Advance(1) } } // isExponentLetter reports whether a rune introduces an exponent, which depends // on the base: a hexadecimal literal uses p, because e is one of its digits. func isExponentLetter(r rune, hexadecimal bool) bool { if hexadecimal { return r == 'p' || r == 'P' } return r == 'e' || r == 'E' } // isDecimalDigit reports whether a rune may appear in a decimal literal after // its first digit. An underscore may, and "underscores may repeat or trail". func isDecimalDigit(r rune) bool { return syntax.IsDigit(r) || r == '_' } // isHexDigit reports whether a rune may appear in a hexadecimal literal. func isHexDigit(r rune) bool { return isDecimalDigit(r) || r >= 'a' && r <= 'f' || r >= 'A' && r <= 'F' } // isOctalDigit reports whether a rune may appear in an octal literal. func isOctalDigit(r rune) bool { return r >= '0' && r <= '7' || r == '_' } // isBinaryDigit reports whether a rune may appear in a binary literal. func isBinaryDigit(r rune) bool { return r == '0' || r == '1' || r == '_' } // --- words ------------------------------------------------------------------ // bangKeywords are the two keywords that end in an exclamation mark. // // The grammar lists try! and guard! among the keywords, and ! is an operator // rune — so without this the mark would be coloured as an operator hanging off // the end of a keyword, which is not what the language sees there. var bangKeywords = map[string]bool{"try": true, "guard": true} // takeWord colours an identifier, deciding what kind of thing it is from the // word itself and from the rune that follows it. func takeWord(s *syntax.LineScanner) { start := s.Pos() advanceWhile(s, syntax.IsWordRune) word := wordAt(s, start) if bangKeywords[word] && s.Peek(0) == '!' { s.Advance(1) s.Emit(start, s.Pos(), syntax.ClassKeyword) return } if isLabel(s, word) { s.Advance(1) // the ~ s.Emit(start, s.Pos(), syntax.ClassAttribute) return } s.Emit(start, s.Pos(), classOfWord(word, s.Peek(0))) } // takeMember colours the name after a dot: a field, or a method. // // It is takeWord without the keyword table, because MoonBit's dot-identifiers // "use the identifier case rules without consulting the keyword table, so .if // is valid". A record with a field called `type` is ordinary MoonBit, and // colouring that field as a keyword would be a claim about the language that // the language contradicts. func takeMember(s *syntax.LineScanner) { start := s.Pos() advanceWhile(s, syntax.IsWordRune) s.Emit(start, s.Pos(), classOfMember(wordAt(s, start), s.Peek(0))) } // wordAt returns the word running from start to the scanner's position. func wordAt(s *syntax.LineScanner, start int) string { var b strings.Builder for at := start; at < s.Pos(); at++ { b.WriteRune(s.Peek(at - s.Pos())) } return b.String() } // isLabel reports whether the word just consumed is a labelled argument's name, // which is to say whether a tilde touches it. // // The tilde is MoonBit's alone: it appears in no operator the language has, so // a tilde against the end of a name can only be a label. The two exclusions are // the grammar's own — "ASCII-uppercase identifiers and keywords cannot form // labels" — and they matter, because without the first, Foo~ in a piece of // half-typed code would colour a type as a label. func isLabel(s *syntax.LineScanner, word string) bool { if s.Peek(0) != '~' || word == "" { return false } if startsUpperCase(word) { return false } _, reserved := knownWords[word] return !reserved } // classOfWord decides what a word is, given the rune that follows it. // // The order is the design, and MoonBit lets it be shorter than any other // scanner in this family. A word the language names is what the language says // it is. After that comes the case rule — and in MoonBit that is a *lexical* // rule rather than a convention: a uident "begins with an ASCII uppercase // letter", and only a type, a trait or an enum constructor may be spelt that // way. So there is no table of built-in types here, and there does not need to // be: Int, StringBuilder and a type somebody wrote this morning are all // capitalised, and all coloured by the same line. // // What that costs is that a constructor of your own enum is coloured as a type. // Nothing in the syntax separates Circle(1.0) from a type applied to arguments, // and inventing a separation would mean being wrong in both directions instead // of one. func classOfWord(word string, next rune) syntax.Class { if class, known := knownWords[word]; known { return class } if startsUpperCase(word) { return syntax.ClassType } if next == '(' { return syntax.ClassFunction } return syntax.ClassIdentifier } // classOfMember decides what the name after a dot is. It is classOfWord with // the keyword table left out; see takeMember. func classOfMember(word string, next rune) syntax.Class { if startsUpperCase(word) { return syntax.ClassType } if next == '(' { return syntax.ClassFunction } return syntax.ClassIdentifier } // startsUpperCase reports whether a word begins with an ASCII capital, which is // what makes it a uident. func startsUpperCase(word string) bool { return word != "" && word[0] >= 'A' && word[0] <= 'Z' } // knownWords is every word the language itself names, and what each one is. // // It is one table rather than three because it answers one question. The three // groups below are kept apart only so that each can carry the reasoning that // belongs to it. var knownWords = merge( classify(syntax.ClassKeyword, keywords), classify(syntax.ClassConstant, constants), classify(syntax.ClassBuiltin, builtinValues), ) // keywords are the words MoonBit reserves, taken from the grammar's keyword // production. // // true and false are in it there and are not here: they are keywords to the // lexer and values to the reader, and every editor in this family colours them // as constants. try! and guard! are not here either — the mark is glued on by // takeWord, because a table cannot hold a word whose last rune is an operator. // // package is here although in a .mbt file it is only a *reserved* word, which // the lexer treats as an identifier and warns about. It is a real keyword in // the .mbti interface files this editor also colours, and in a .mbt file // colouring it says exactly what the compiler is about to: this word is not // yours to use. // // The rest of the reserved list — move, ref, static, unsafe, await, and the // forty others — is deliberately absent. Those are identifiers that earn a // warning, and a scanner that coloured them as keywords would be telling a // reader they cannot write `let ref = 1` when they can. var keywords = words( "and", "as", "async", "break", "catch", "const", "continue", "declare", "defer", "derive", "else", "enum", "enumview", "extend", "extenum", "extern", "fn", "for", "guard", "if", "impl", "import", "in", "is", "let", "letrec", "lexscan", "loop", "match", "mut", "nobreak", "nocancel", "noraise", "package", "priv", "proof_assert", "proof_let", "pub", "raise", "readonly", "return", "struct", "suberror", "test", "throw", "trait", "try", "type", "using", "where", "while", "with", ) // constants are the values a reader meets as the language's own. // // None, Some, Ok and Err belong to Option and Result rather than to the // language, but a reader meets them everywhere and reads them as built in, the // same argument Turbo Rust records for the same four names. var constants = words("true", "false", "None", "Some", "Ok", "Err") // builtinValues are the lower-case names the prelude puts in scope without an // import, read out of moonbitlang/core/prelude rather than remembered. // // The prelude's deprecated names — dump, not, tap, then, to_repr — are left // out on purpose: colouring them as builtins would present as the language's // own four things it is trying to retire. There is no print here for the same // kind of reason, and it is the one worth stating: MoonBit has println and has // never had print, so a table written from habit would have coloured a name // that does not exist. var builtinValues = words( "abort", "assert_eq", "assert_false", "assert_not_eq", "assert_true", "compare", "debug", "debug_assert", "debug_inspect", "fail", "hash", "ignore", "inspect", "json_inspect", "null", "panic", "physical_equal", "println", "repr", ) // words gathers a group of them, which reads better at the call sites above // than a slice literal does. func words(list ...string) []string { return list } // classify pairs every word in a group with the class it belongs to. func classify(class syntax.Class, list []string) map[string]syntax.Class { out := make(map[string]syntax.Class, len(list)) for _, word := range list { out[word] = class } return out } // merge folds the groups into one table. An earlier group wins a word a later // one repeats, which is what keeps a keyword a keyword. func merge(groups ...map[string]syntax.Class) map[string]syntax.Class { out := map[string]syntax.Class{} for _, group := range groups { for word, class := range group { if _, taken := out[word]; !taken { out[word] = class } } } return out }