| 📦 Turbo MoonBit cc1f595 k33g 22h ago | 1 | package moonbitlang |
| 2 | |
| 3 | import "rickub.com/turbo-editors/turbo-core/syntax" |
| 4 | |
| 5 | // carry is what a line of MoonBit leaves open for the next one — and in this |
| 6 | // language, nothing does. |
| 7 | // |
| 8 | // Every other editor in this family threads real state through its scanner: Go |
| 9 | // and Rust carry a block-comment depth, Rust carries a raw string's delimiter, |
| 10 | // Python carries which quote opened a triple-quoted literal. MoonBit needs |
| 11 | // none of it, and that is a property of the language rather than a shortcut: |
| 12 | // |
| 13 | // - There is no block comment. The grammar says so in as many words: "MoonBit |
| 14 | // has no block-comment form." // runs to the end of the line, and /// is a |
| 15 | // doc comment that does the same. |
| 16 | // - A newline before a closing quote is an *unterminated literal* error, for |
| 17 | // strings, bytes, regexes, characters and byte characters alike. No literal |
| 18 | // may reach the next line, so none of them can be carried onto it. |
| 19 | // - A multi-line string is not one literal spanning lines. It is a run of |
| 20 | // lines each prefixed #| or $|, each complete in itself, joined afterwards. |
| 21 | // - An attribute is explicitly one line: "everything through the next newline |
| 22 | // is the raw payload". |
| 23 | // |
| 24 | // So the type is empty, and it is a named type rather than struct{} written |
| 25 | // inline so that this comment has somewhere to live. If MoonBit ever grows a |
| 26 | // construct that crosses a line break, this is the type that gains a field and |
| 27 | // scanLine is where it would be threaded. |
| 28 | type carry struct{} |
| 29 | |
| 30 | // Highlight colours MoonBit source. |
| 31 | // |
| 32 | // It is written against syntax.LineScanner, a line at a time. MoonBit has no |
| 33 | // tokeniser in the Go standard library the way Go does, so this is a scanner in |
| 34 | // the same style as the ones turbo-core ships for TOML, Markdown and shell. |
| 35 | // |
| 36 | // It is deliberately tolerant of broken input: source under the cursor is |
| 37 | // invalid most of the time it is being typed, and a highlighter that gives up |
| 38 | // is a highlighter that flickers off. |
| 39 | // |
| 40 | // spans := moonbitlang.Highlight("fn main {\n println(\"hi\")\n}\n") |
| 41 | // // spans[0][0] covers "fn" with syntax.ClassKeyword |
| 42 | func Highlight(src string) [][]syntax.Span { |
| 43 | return syntax.ScanLines(src, scanLine) |
| 44 | } |
| 45 | |
| 46 | // scanLine colours one line. The carry is threaded because ScanLines asks for |
| 47 | // it, and is returned untouched because nothing in MoonBit crosses a line. |
| 48 | func scanLine(line []rune, open carry) ([]syntax.Span, carry) { |
| 49 | s := syntax.NewLineScanner(line) |
| 50 | for !s.AtEnd() { |
| 51 | scanToken(s) |
| 52 | } |
| 53 | return s.Spans(), open |
| 54 | } |
| 55 | |
| 56 | // scanToken colours whatever starts at the scanner's position. |
| 57 | // |
| 58 | // The order of the cases is the design, and three of them are load-bearing: |
| 59 | // |
| 60 | // - "//" is tested before the operators, because / is an operator rune. |
| 61 | // - A multi-line string line is tested before an attribute, because both |
| 62 | // begin with #, and what tells them apart is the rune after it. |
| 63 | // - A literal is tested before a word, because b" and re" begin with letters: |
| 64 | // without this, b"bytes" would be an identifier followed by a string. |
| 65 | func scanToken(s *syntax.LineScanner) { |
| 66 | r := s.Peek(0) |
| 67 | |
| 68 | switch { |
| 69 | case r == ' ' || r == '\t': |
| 70 | s.SkipSpaces() |
| 71 | case s.HasPrefix(0, "//"): |
| 72 | // /// is a doc comment and // is an ordinary one. They are the same |
| 73 | // colour because turbo-core's Class set has one comment class, and |
| 74 | // that set is closed on purpose — a theme colours every language an |
| 75 | // editor will ever learn. |
| 76 | s.TakeRest(syntax.ClassComment) |
| 77 | case isMultilineStringStart(s): |
| 78 | takeMultilineStringLine(s) |
| 79 | case isAttributeStart(s): |
| 80 | // The grammar hands the whole rest of the line to the attribute: after |
| 81 | // the dotted name, "everything through the next newline is the raw |
| 82 | // payload". Colouring less than the line would be inventing a |
| 83 | // structure the lexer does not have. |
| 84 | s.TakeRest(syntax.ClassAttribute) |
| 85 | case isPackageStart(s): |
| 86 | takePackageName(s) |
| 87 | case isLiteralStart(s): |
| 88 | takeLiteral(s) |
| 89 | case syntax.IsDigit(r): |
| 90 | // A leading dot is never a number in MoonBit: the grammar requires a |
| 91 | // digit before the point, so .5 is not a literal and .0 is a tuple |
| 92 | // accessor. That is why this case asks for a digit and nothing else. |
| 93 | takeNumber(s) |
| 94 | case r == '.': |
| 95 | takeDot(s) |
| 96 | case syntax.IsLetter(r) || r == '_': |
| 97 | takeWord(s) |
| 98 | case syntax.IsOperatorRune(r): |
| 99 | s.TakeWhile(syntax.ClassOperator, syntax.IsOperatorRune) |
| 100 | case syntax.IsPunctuationRune(r): |
| 101 | s.Take(1, syntax.ClassPunctuation) |
| 102 | default: |
| 103 | // A rune nothing here claims — a CJK letter in an identifier, which |
| 104 | // MoonBit allows and turbo-core's ASCII rune predicates do not — is |
| 105 | // stepped over uncoloured rather than guessed at. |
| 106 | s.Advance(1) |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | // --- comments' neighbours: attributes and multi-line strings ---------------- |
| 111 | |
| 112 | // isAttributeStart reports whether an attribute opens at the scanner's |
| 113 | // position. |
| 114 | // |
| 115 | // An attribute name must begin with a letter or an underscore, which is the |
| 116 | // whole of what separates #deprecated from the #| that opens a raw multi-line |
| 117 | // string line. |
| 118 | func isAttributeStart(s *syntax.LineScanner) bool { |
| 119 | if s.Peek(0) != '#' { |
| 120 | return false |
| 121 | } |
| 122 | next := s.Peek(1) |
| 123 | return syntax.IsLetter(next) || next == '_' |
| 124 | } |
| 125 | |
| 126 | // isMultilineStringStart reports whether a multi-line string line opens at the |
| 127 | // scanner's position. |
| 128 | // |
| 129 | // #| is a raw line and $| an interpolated one. Neither is a literal that |
| 130 | // continues: the lines are separate tokens which the compiler joins with a |
| 131 | // newline afterwards, which is exactly why this scanner needs no carry. |
| 132 | func isMultilineStringStart(s *syntax.LineScanner) bool { |
| 133 | return (s.Peek(0) == '#' || s.Peek(0) == '$') && s.Peek(1) == '|' |
| 134 | } |
| 135 | |
| 136 | // takeMultilineStringLine colours one #| or $| line. |
| 137 | // |
| 138 | // The two-rune prefix is punctuation rather than string, because it is not part |
| 139 | // of the value: it says "this line is text", and the text is what follows it. |
| 140 | func takeMultilineStringLine(s *syntax.LineScanner) { |
| 141 | s.Take(2, syntax.ClassPunctuation) |
| 142 | s.TakeRest(syntax.ClassString) |
| 143 | } |
| 144 | |
| 145 | // --- package names ---------------------------------------------------------- |
| 146 | |
| 147 | // isPackageStart reports whether a package name opens at the scanner's |
| 148 | // position: @ followed by the first rune of a package part. |
| 149 | func isPackageStart(s *syntax.LineScanner) bool { |
| 150 | if s.Peek(0) != '@' { |
| 151 | return false |
| 152 | } |
| 153 | next := s.Peek(1) |
| 154 | return syntax.IsLetter(next) || next == '_' |
| 155 | } |
| 156 | |
| 157 | // takePackageName colours @json, @moonbitlang/core/builtin and the like. |
| 158 | // |
| 159 | // It is one span including the @, because that is one token: the grammar says |
| 160 | // "the leading @, slashes, and parts must be adjacent". ClassType is the |
| 161 | // closest of the seventeen classes — a package qualifier names a thing rather |
| 162 | // than holding a value, and the reading it has to be saved from is the one |
| 163 | // where @json.parse looks like a local variable called json. |
| 164 | // |
| 165 | // A slash or a hyphen is only taken when a name follows it, so @a/2 stops at |
| 166 | // the slash and @a - b stops at the space, rather than swallowing an operator |
| 167 | // because it happened to touch a package name. |
| 168 | func takePackageName(s *syntax.LineScanner) { |
| 169 | start := s.Pos() |
| 170 | s.Advance(1) // the @ |
| 171 | |
| 172 | for !s.AtEnd() { |
| 173 | switch r := s.Peek(0); { |
| 174 | case syntax.IsWordRune(r): |
| 175 | s.Advance(1) |
| 176 | case r == '/' && (syntax.IsLetter(s.Peek(1)) || s.Peek(1) == '_'): |
| 177 | s.Advance(1) |
| 178 | case r == '-' && syntax.IsWordRune(s.Peek(1)): |
| 179 | s.Advance(1) |
| 180 | default: |
| 181 | s.Emit(start, s.Pos(), syntax.ClassType) |
| 182 | return |
| 183 | } |
| 184 | } |
| 185 | s.Emit(start, s.Pos(), syntax.ClassType) |
| 186 | } |
| 187 | |
| 188 | // --- what follows a dot ----------------------------------------------------- |
| 189 | |
| 190 | // takeDot colours a dot and whatever the language says belongs with it. |
| 191 | // |
| 192 | // Three things begin with one, and telling them apart is what keeps 1..=2 from |
| 193 | // being read as the number 1. followed by =2: |
| 194 | // |
| 195 | // - a run of dots is a range operator — .. ..= ..< ... |
| 196 | // - a dot with digits after it is a tuple accessor, pair.0 |
| 197 | // - a dot with a name after it is a field or a method, xs.length() |
| 198 | // |
| 199 | // The third case is the reason this is a function rather than a punctuation |
| 200 | // rune. A dot-identifier "uses the identifier case rules without consulting the |
| 201 | // keyword table, so .if is valid" — so the name after a dot must not be looked |
| 202 | // up among the keywords, and takeMember is the version of takeWord that does |
| 203 | // not. |
| 204 | func takeDot(s *syntax.LineScanner) { |
| 205 | switch next := s.Peek(1); { |
| 206 | case next == '.': |
| 207 | s.TakeWhile(syntax.ClassOperator, func(r rune) bool { return r == '.' }) |
| 208 | case syntax.IsDigit(next): |
| 209 | s.Take(1, syntax.ClassPunctuation) |
| 210 | s.TakeWhile(syntax.ClassNumber, syntax.IsDigit) |
| 211 | case syntax.IsLetter(next) || next == '_': |
| 212 | s.Take(1, syntax.ClassPunctuation) |
| 213 | takeMember(s) |
| 214 | default: |
| 215 | s.Take(1, syntax.ClassPunctuation) |
| 216 | } |
| 217 | } |