package moonbitlang import "rickub.com/turbo-editors/turbo-core/syntax" // carry is what a line of MoonBit leaves open for the next one — and in this // language, nothing does. // // Every other editor in this family threads real state through its scanner: Go // and Rust carry a block-comment depth, Rust carries a raw string's delimiter, // Python carries which quote opened a triple-quoted literal. MoonBit needs // none of it, and that is a property of the language rather than a shortcut: // // - There is no block comment. The grammar says so in as many words: "MoonBit // has no block-comment form." // runs to the end of the line, and /// is a // doc comment that does the same. // - A newline before a closing quote is an *unterminated literal* error, for // strings, bytes, regexes, characters and byte characters alike. No literal // may reach the next line, so none of them can be carried onto it. // - A multi-line string is not one literal spanning lines. It is a run of // lines each prefixed #| or $|, each complete in itself, joined afterwards. // - An attribute is explicitly one line: "everything through the next newline // is the raw payload". // // So the type is empty, and it is a named type rather than struct{} written // inline so that this comment has somewhere to live. If MoonBit ever grows a // construct that crosses a line break, this is the type that gains a field and // scanLine is where it would be threaded. type carry struct{} // Highlight colours MoonBit source. // // It is written against syntax.LineScanner, a line at a time. MoonBit has no // tokeniser in the Go standard library the way Go does, so this is a scanner in // the same style as the ones turbo-core ships for TOML, Markdown and shell. // // It is deliberately tolerant of broken input: source under the cursor is // invalid most of the time it is being typed, and a highlighter that gives up // is a highlighter that flickers off. // // spans := moonbitlang.Highlight("fn main {\n println(\"hi\")\n}\n") // // spans[0][0] covers "fn" with syntax.ClassKeyword func Highlight(src string) [][]syntax.Span { return syntax.ScanLines(src, scanLine) } // scanLine colours one line. The carry is threaded because ScanLines asks for // it, and is returned untouched because nothing in MoonBit crosses a line. func scanLine(line []rune, open carry) ([]syntax.Span, carry) { s := syntax.NewLineScanner(line) for !s.AtEnd() { scanToken(s) } return s.Spans(), open } // scanToken colours whatever starts at the scanner's position. // // The order of the cases is the design, and three of them are load-bearing: // // - "//" is tested before the operators, because / is an operator rune. // - A multi-line string line is tested before an attribute, because both // begin with #, and what tells them apart is the rune after it. // - A literal is tested before a word, because b" and re" begin with letters: // without this, b"bytes" would be an identifier followed by a string. func scanToken(s *syntax.LineScanner) { r := s.Peek(0) switch { case r == ' ' || r == '\t': s.SkipSpaces() case s.HasPrefix(0, "//"): // /// is a doc comment and // is an ordinary one. They are the same // colour because turbo-core's Class set has one comment class, and // that set is closed on purpose — a theme colours every language an // editor will ever learn. s.TakeRest(syntax.ClassComment) case isMultilineStringStart(s): takeMultilineStringLine(s) case isAttributeStart(s): // The grammar hands the whole rest of the line to the attribute: after // the dotted name, "everything through the next newline is the raw // payload". Colouring less than the line would be inventing a // structure the lexer does not have. s.TakeRest(syntax.ClassAttribute) case isPackageStart(s): takePackageName(s) case isLiteralStart(s): takeLiteral(s) case syntax.IsDigit(r): // A leading dot is never a number in MoonBit: the grammar requires a // digit before the point, so .5 is not a literal and .0 is a tuple // accessor. That is why this case asks for a digit and nothing else. takeNumber(s) case r == '.': takeDot(s) case syntax.IsLetter(r) || r == '_': takeWord(s) case syntax.IsOperatorRune(r): s.TakeWhile(syntax.ClassOperator, syntax.IsOperatorRune) case syntax.IsPunctuationRune(r): s.Take(1, syntax.ClassPunctuation) default: // A rune nothing here claims — a CJK letter in an identifier, which // MoonBit allows and turbo-core's ASCII rune predicates do not — is // stepped over uncoloured rather than guessed at. s.Advance(1) } } // --- comments' neighbours: attributes and multi-line strings ---------------- // isAttributeStart reports whether an attribute opens at the scanner's // position. // // An attribute name must begin with a letter or an underscore, which is the // whole of what separates #deprecated from the #| that opens a raw multi-line // string line. func isAttributeStart(s *syntax.LineScanner) bool { if s.Peek(0) != '#' { return false } next := s.Peek(1) return syntax.IsLetter(next) || next == '_' } // isMultilineStringStart reports whether a multi-line string line opens at the // scanner's position. // // #| is a raw line and $| an interpolated one. Neither is a literal that // continues: the lines are separate tokens which the compiler joins with a // newline afterwards, which is exactly why this scanner needs no carry. func isMultilineStringStart(s *syntax.LineScanner) bool { return (s.Peek(0) == '#' || s.Peek(0) == '$') && s.Peek(1) == '|' } // takeMultilineStringLine colours one #| or $| line. // // The two-rune prefix is punctuation rather than string, because it is not part // of the value: it says "this line is text", and the text is what follows it. func takeMultilineStringLine(s *syntax.LineScanner) { s.Take(2, syntax.ClassPunctuation) s.TakeRest(syntax.ClassString) } // --- package names ---------------------------------------------------------- // isPackageStart reports whether a package name opens at the scanner's // position: @ followed by the first rune of a package part. func isPackageStart(s *syntax.LineScanner) bool { if s.Peek(0) != '@' { return false } next := s.Peek(1) return syntax.IsLetter(next) || next == '_' } // takePackageName colours @json, @moonbitlang/core/builtin and the like. // // It is one span including the @, because that is one token: the grammar says // "the leading @, slashes, and parts must be adjacent". ClassType is the // closest of the seventeen classes — a package qualifier names a thing rather // than holding a value, and the reading it has to be saved from is the one // where @json.parse looks like a local variable called json. // // A slash or a hyphen is only taken when a name follows it, so @a/2 stops at // the slash and @a - b stops at the space, rather than swallowing an operator // because it happened to touch a package name. func takePackageName(s *syntax.LineScanner) { start := s.Pos() s.Advance(1) // the @ for !s.AtEnd() { switch r := s.Peek(0); { case syntax.IsWordRune(r): s.Advance(1) case r == '/' && (syntax.IsLetter(s.Peek(1)) || s.Peek(1) == '_'): s.Advance(1) case r == '-' && syntax.IsWordRune(s.Peek(1)): s.Advance(1) default: s.Emit(start, s.Pos(), syntax.ClassType) return } } s.Emit(start, s.Pos(), syntax.ClassType) } // --- what follows a dot ----------------------------------------------------- // takeDot colours a dot and whatever the language says belongs with it. // // Three things begin with one, and telling them apart is what keeps 1..=2 from // being read as the number 1. followed by =2: // // - a run of dots is a range operator — .. ..= ..< ... // - a dot with digits after it is a tuple accessor, pair.0 // - a dot with a name after it is a field or a method, xs.length() // // The third case is the reason this is a function rather than a punctuation // rune. A dot-identifier "uses the identifier case rules without consulting the // keyword table, so .if is valid" — so the name after a dot must not be looked // up among the keywords, and takeMember is the version of takeWord that does // not. func takeDot(s *syntax.LineScanner) { switch next := s.Peek(1); { case next == '.': s.TakeWhile(syntax.ClassOperator, func(r rune) bool { return r == '.' }) case syntax.IsDigit(next): s.Take(1, syntax.ClassPunctuation) s.TakeWhile(syntax.ClassNumber, syntax.IsDigit) case syntax.IsLetter(next) || next == '_': s.Take(1, syntax.ClassPunctuation) takeMember(s) default: s.Take(1, syntax.ClassPunctuation) } }