| 📦 Turbo Golo d710c1b k33g 22h ago | 1 | package gololang |
| 2 | |
| 3 | // Numbers and words: what a run of digits or letters turns out to be. |
| 4 | |
| 5 | import ( |
| 6 | "strings" |
| 7 | "unicode" |
| 8 | |
| 9 | "rickub.com/turbo-editors/turbo-core/syntax" |
| 10 | ) |
| 11 | |
| 12 | // --- numbers ---------------------------------------------------------------- |
| 13 | |
| 14 | // takeNumber colours a numeric literal the way lexer.go's readNumber reads |
| 15 | // one: digits, then a point and more digits if a digit follows the point, then |
| 16 | // an exponent with an optional sign, then the L that makes a long or the F or |
| 17 | // f that makes a float. |
| 18 | // |
| 19 | // The point is only part of the number when a digit follows it. That is what |
| 20 | // keeps 1..3 a number and a range rather than the double 1. and a stray .3, |
| 21 | // and it is the lexer's own test — `l.ch == '.' && isDigit(l.peekChar())`. |
| 22 | // |
| 23 | // There is no hexadecimal, no binary, no octal and no digit separator, because |
| 24 | // the lexer has none: 0xFF is the number 0 followed by the name xFF, and 1_000 |
| 25 | // is 1 followed by the name _000. Colouring either as one number would be |
| 26 | // inventing a literal the interpreter will reject. |
| 27 | // |
| 28 | // The whole literal is one span, so every step here advances the scanner |
| 29 | // without colouring and the single Emit at the end covers what they consumed. |
| 30 | func takeNumber(s *syntax.LineScanner) { |
| 31 | start := s.Pos() |
| 32 | advanceWhile(s, syntax.IsDigit) |
| 33 | |
| 34 | if s.Peek(0) == '.' && syntax.IsDigit(s.Peek(1)) { |
| 35 | s.Advance(1) |
| 36 | advanceWhile(s, syntax.IsDigit) |
| 37 | } |
| 38 | takeExponent(s) |
| 39 | takeNumberSuffix(s) |
| 40 | |
| 41 | s.Emit(start, s.Pos(), syntax.ClassNumber) |
| 42 | } |
| 43 | |
| 44 | // takeExponent consumes e or E, an optional sign, and the digits after them. |
| 45 | // |
| 46 | // The digits may be none. The lexer reads 1e as a float and leaves the parser |
| 47 | // to complain, and this scanner colours what the lexer reads. |
| 48 | func takeExponent(s *syntax.LineScanner) { |
| 49 | if s.Peek(0) != 'e' && s.Peek(0) != 'E' { |
| 50 | return |
| 51 | } |
| 52 | s.Advance(1) |
| 53 | if s.Peek(0) == '+' || s.Peek(0) == '-' { |
| 54 | s.Advance(1) |
| 55 | } |
| 56 | advanceWhile(s, syntax.IsDigit) |
| 57 | } |
| 58 | |
| 59 | // takeNumberSuffix consumes the L of a long and the F or f of a float, in |
| 60 | // that order, when the literal ends in them. 42L is a long; 3.14F and 2.0f are |
| 61 | // floats; the lexer accepts both cases of the F and only the upper case of |
| 62 | // the L. |
| 63 | func takeNumberSuffix(s *syntax.LineScanner) { |
| 64 | if s.Peek(0) == 'L' { |
| 65 | s.Advance(1) |
| 66 | } |
| 67 | if s.Peek(0) == 'F' || s.Peek(0) == 'f' { |
| 68 | s.Advance(1) |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | // advanceWhile steps over runes that match, without colouring any of them. It |
| 73 | // is what a construct emitted as a single span uses in place of TakeWhile, |
| 74 | // which would colour each run it consumed and leave the Emit overlapping it. |
| 75 | func advanceWhile(s *syntax.LineScanner, matches func(rune) bool) { |
| 76 | for !s.AtEnd() && matches(s.Peek(0)) { |
| 77 | s.Advance(1) |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | // --- words ------------------------------------------------------------------ |
| 82 | |
| 83 | // isIdentifierStart reports whether a rune may begin a name, by the lexer's |
| 84 | // own isLetter: any Unicode letter or mark, an underscore, or an emoji. |
| 85 | // |
| 86 | // This is wider than turbo-core's ASCII IsLetter on purpose. Golo lets you |
| 87 | // write `let 😀 = 1` and `function 🚀launch = { … }`, and a scanner that left |
| 88 | // those uncoloured would be telling a reader they are not names when the |
| 89 | // interpreter says they are. |
| 90 | func isIdentifierStart(r rune) bool { |
| 91 | return unicode.IsLetter(r) || unicode.IsMark(r) || r == '_' || isEmoji(r) |
| 92 | } |
| 93 | |
| 94 | // isWordRune reports whether a rune may continue a name: whatever may begin |
| 95 | // one, or a digit. |
| 96 | func isWordRune(r rune) bool { |
| 97 | return isIdentifierStart(r) || syntax.IsDigit(r) |
| 98 | } |
| 99 | |
| 100 | // emojiBlocks are the four Unicode blocks the lexer admits into a name, each |
| 101 | // as its first and last rune. They are the lexer's own, copied rather than |
| 102 | // widened: the general symbol blocks are left out there because they hold the |
| 103 | // operators. |
| 104 | var emojiBlocks = [][2]rune{ |
| 105 | {0x1F600, 0x1F64F}, // emoticons |
| 106 | {0x1F300, 0x1F5FF}, // miscellaneous symbols and pictographs |
| 107 | {0x1F680, 0x1F6FF}, // transport and map symbols |
| 108 | {0x1F900, 0x1F9FF}, // supplemental symbols and pictographs |
| 109 | } |
| 110 | |
| 111 | // isEmoji reports whether a rune is in one of the blocks the lexer admits into |
| 112 | // a name. |
| 113 | func isEmoji(r rune) bool { |
| 114 | for _, block := range emojiBlocks { |
| 115 | if r >= block[0] && r <= block[1] { |
| 116 | return true |
| 117 | } |
| 118 | } |
| 119 | return false |
| 120 | } |
| 121 | |
| 122 | // takeWord colours a name, deciding what kind of thing it is from the word |
| 123 | // itself and from the rune that follows it — and, for two keywords, colours |
| 124 | // the name that follows *them*, because that name means something only there. |
| 125 | func takeWord(s *syntax.LineScanner) { |
| 126 | start := s.Pos() |
| 127 | advanceWhile(s, isWordRune) |
| 128 | word := wordAt(s, start) |
| 129 | s.Emit(start, s.Pos(), classOfWord(word, s.Peek(0))) |
| 130 | |
| 131 | switch word { |
| 132 | case "module", "import": |
| 133 | takeModulePath(s) |
| 134 | case "function": |
| 135 | takeDeclaredFunctionName(s) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | // takeModulePath colours the dotted name after module or import as one span: |
| 140 | // hello.World, gololang.Errors, java.util.List. |
| 141 | // |
| 142 | // It is one span because it is one name — a module has no parts a program can |
| 143 | // take apart — and ClassType is the nearest of the seventeen classes: a module |
| 144 | // path names a thing rather than holding a value, and the reading it has to be |
| 145 | // saved from is the one where gololang.Errors looks like a variable called |
| 146 | // gololang with something done to it. |
| 147 | // |
| 148 | // A dot is only taken when a name follows it, so `import a.` at the end of a |
| 149 | // half-typed line stops before the dot and leaves it as punctuation. |
| 150 | func takeModulePath(s *syntax.LineScanner) { |
| 151 | s.SkipSpaces() |
| 152 | if !isIdentifierStart(s.Peek(0)) { |
| 153 | return |
| 154 | } |
| 155 | |
| 156 | start := s.Pos() |
| 157 | advanceWhile(s, isWordRune) |
| 158 | for s.Peek(0) == '.' && isIdentifierStart(s.Peek(1)) { |
| 159 | s.Advance(1) |
| 160 | advanceWhile(s, isWordRune) |
| 161 | } |
| 162 | s.Emit(start, s.Pos(), syntax.ClassType) |
| 163 | } |
| 164 | |
| 165 | // takeDeclaredFunctionName colours the name after the function keyword as a |
| 166 | // function. |
| 167 | // |
| 168 | // Everywhere else a name is a function because a parenthesis follows it, and |
| 169 | // a declaration is the one place that is not true: `function main = |args|` |
| 170 | // has the name followed by an equals sign. Without this, every function a |
| 171 | // file declares would be coloured as an ordinary variable at the one place a |
| 172 | // reader looks for it. |
| 173 | func takeDeclaredFunctionName(s *syntax.LineScanner) { |
| 174 | s.SkipSpaces() |
| 175 | if !isIdentifierStart(s.Peek(0)) { |
| 176 | return |
| 177 | } |
| 178 | |
| 179 | start := s.Pos() |
| 180 | advanceWhile(s, isWordRune) |
| 181 | s.Emit(start, s.Pos(), syntax.ClassFunction) |
| 182 | } |
| 183 | |
| 184 | // wordAt returns the word running from start to the scanner's position. |
| 185 | func wordAt(s *syntax.LineScanner, start int) string { |
| 186 | var b strings.Builder |
| 187 | for at := start; at < s.Pos(); at++ { |
| 188 | b.WriteRune(s.Peek(at - s.Pos())) |
| 189 | } |
| 190 | return b.String() |
| 191 | } |
| 192 | |
| 193 | // classOfWord decides what a word is, given the rune that follows it. |
| 194 | // |
| 195 | // The order is the design. A word the language names is what the language |
| 196 | // says it is: a keyword, one of the three literal constants, or one of the |
| 197 | // interpreter's built-in functions. After that comes the case rule, which in |
| 198 | // Golo is a convention rather than a lexical fact: structs, unions and their |
| 199 | // variants are capitalised by everybody — Point, Shape, Circle, Some, None — |
| 200 | // and nothing else customarily is, so a capitalised word is coloured as a |
| 201 | // type. A lower-case word followed by a parenthesis is a call, and anything |
| 202 | // else is a name. |
| 203 | // |
| 204 | // What the case rule costs is that a variant's constructor is coloured as a |
| 205 | // type — Circle(1.0) and Result_Failure("no") look like types applied to |
| 206 | // arguments — and a capitalised variable, which Golo permits, is coloured as |
| 207 | // one too. Nothing in the syntax separates them, and inventing a separation |
| 208 | // would mean being wrong in both directions instead of one. |
| 209 | func classOfWord(word string, next rune) syntax.Class { |
| 210 | if class, known := knownWords[word]; known { |
| 211 | return class |
| 212 | } |
| 213 | if startsUpperCase(word) { |
| 214 | return syntax.ClassType |
| 215 | } |
| 216 | if next == '(' { |
| 217 | return syntax.ClassFunction |
| 218 | } |
| 219 | return syntax.ClassIdentifier |
| 220 | } |
| 221 | |
| 222 | // startsUpperCase reports whether a word begins with an ASCII capital, which |
| 223 | // is what the convention means by a type's name. |
| 224 | func startsUpperCase(word string) bool { |
| 225 | return word != "" && word[0] >= 'A' && word[0] <= 'Z' |
| 226 | } |
| 227 | |
| 228 | // knownWords is every word the language itself names, and what each one is. |
| 229 | // |
| 230 | // It is one table rather than three because it answers one question. The three |
| 231 | // groups below are kept apart only so that each can carry the reasoning that |
| 232 | // belongs to it. |
| 233 | var knownWords = merge( |
| 234 | classify(syntax.ClassKeyword, keywords), |
| 235 | classify(syntax.ClassConstant, constants), |
| 236 | classify(syntax.ClassBuiltin, builtinFunctions), |
| 237 | ) |
| 238 | |
| 239 | // keywords are the words Golo reserves, taken from token/token.go's keyword |
| 240 | // table in GoloScript — every entry of it except the three literal values, |
| 241 | // which are constants below. |
| 242 | // |
| 243 | // The word operators are here as keywords: and, or, not, is, isnt, oftype and |
| 244 | // orIfNull are reserved words that happen to compute something, and a reader |
| 245 | // meets them as words. There is no `then`-less if or `elseif`; `else if` is |
| 246 | // two keywords. |
| 247 | var keywords = words( |
| 248 | "and", "augment", "augmentation", "await", "break", "case", "catch", |
| 249 | "continue", "else", "finally", "for", "foreach", "function", "if", |
| 250 | "import", "in", "is", "isnt", "let", "local", "match", "module", "not", |
| 251 | "oftype", "or", "orIfNull", "otherwise", "return", "spawn", "struct", |
| 252 | "then", "throw", "try", "union", "var", "when", "while", "with", |
| 253 | ) |
| 254 | |
| 255 | // constants are the values a reader meets as the language's own. They are |
| 256 | // keywords to the lexer and values to the reader, and every editor in this |
| 257 | // family colours them as constants. |
| 258 | var constants = words("true", "false", "null") |
| 259 | |
| 260 | // builtinFunctions are the functions the interpreter provides without an |
| 261 | // import, read out of evaluator.BuiltinNames() in GoloScript rather than |
| 262 | // remembered — 162 names, less the five prefixed with a double underscore, |
| 263 | // which are the test runner's own counters and which the language server |
| 264 | // likewise keeps out of its completion list. |
| 265 | // |
| 266 | // Some, None, Ok and Err are deliberately not here. In Golo they are not built |
| 267 | // in: they are the variants of ordinary unions declared in gololang.Errors, |
| 268 | // available only after `import gololang.Errors`, so they take the colour every |
| 269 | // other capitalised name does. DynamicObject *is* here, capital and all, |
| 270 | // because it is a builtin function that happens to be spelt like a type. |
| 271 | var builtinFunctions = words( |
| 272 | "DynamicObject", "abs", "appendFile", "array", "chanClose", "chanReceive", |
| 273 | "chanSend", "channel", "currentAbsDir", "currentDir", "currentTime", |
| 274 | "currentTimeMillis", "currentTimeNano", "dateString", "dateTimeString", |
| 275 | "deleteFile", "escapeJSON", "execCombinedOutput", "execCommand", |
| 276 | "fileExists", "fileInfo", "float", "formatTime", "fromJSON", "getenv", |
| 277 | "head", "httpDelete", "httpDeleteStream", "httpGet", "httpGetStream", |
| 278 | "httpPost", "httpPostStream", "httpPut", "httpPutStream", "httpServe", |
| 279 | "httpStop", "information", "int", "isNotNull", "isNull", "len", "length", |
| 280 | "list", "listDir", "map", "mcpAddResource", "mcpAddTool", "mcpCallTool", |
| 281 | "mcpConnectHTTP", "mcpConnectStdio", "mcpCreateServer", "mcpDisconnect", |
| 282 | "mcpListResources", "mcpListTools", "mcpReadResource", "mcpRunHTTP", |
| 283 | "mcpRunStdio", "mcpStopServer", "mkDir", "mutex", "mutexLock", |
| 284 | "mutexUnlock", "now", "observable", "observableFilter", "observableGet", |
| 285 | "observableMap", "observableOnChange", "observableSet", |
| 286 | "openAIChatCompletion", "openAIChatCompletionStream", |
| 287 | "openAICreateEmbedding", "openAINewClient", "parseTime", "pow", "print", |
| 288 | "println", "push", "raise", "range", "read", "readFile", "readln", |
| 289 | "require", "requireNotNull", "set", "setenv", "sharedGet", "sharedSet", |
| 290 | "sharedState", "sharedUpdate", "sleep", "sqrt", "str", "tail", "template", |
| 291 | "timeString", "toJSON", "tuiClick", "tuiComponentIds", "tuiDisplayWidth", |
| 292 | "tuiEmit", "tuiFocus", "tuiFrame", "tuiGetProp", "tuiHasComponent", |
| 293 | "tuiHide", "tuiIsFocused", "tuiIsVisible", "tuiLoad", "tuiLoadStyle", |
| 294 | "tuiNew", "tuiOff", "tuiOn", "tuiQuit", "tuiRenderMarkdown", "tuiRun", |
| 295 | "tuiSetProp", "tuiShow", "tuiSize", "tuiWheel", "tuiZoneOf", |
| 296 | "tupleFromArray", "type", "uiConfirm", "uiError", "uiGetColorCode", |
| 297 | "uiInfo", "uiMarkdownRender", "uiMarkdownStreamAppend", |
| 298 | "uiMarkdownStreamEnd", "uiMarkdownStreamStart", "uiPrint", |
| 299 | "uiPrintMultiStyle", "uiPrintln", "uiPrompt", "uiPromptMultiline", |
| 300 | "uiPromptPassword", "uiSpinnerError", "uiSpinnerNew", "uiSpinnerSetFrames", |
| 301 | "uiSpinnerSetPrefix", "uiSpinnerSetSuffix", "uiSpinnerStart", |
| 302 | "uiSpinnerStop", "uiSpinnerSuccess", "uiSuccess", "uiWarning", "vector", |
| 303 | "wasmCallNumbers", "wasmCallString", "wasmClose", "wasmHasFunction", |
| 304 | "wasmLoad", "wasmRegisterStringHandler", "wasmShutdown", "writeFile", |
| 305 | ) |
| 306 | |
| 307 | // Builtins returns the interpreter's built-in function names, sorted, so a |
| 308 | // test can hold the table above to what a real golo answers. |
| 309 | func Builtins() []string { |
| 310 | out := make([]string, len(builtinFunctions)) |
| 311 | copy(out, builtinFunctions) |
| 312 | return out |
| 313 | } |
| 314 | |
| 315 | // Keywords returns the reserved words the scanner colours as keywords, so a |
| 316 | // test can hold the table above to what a real golo reserves. |
| 317 | func Keywords() []string { |
| 318 | out := make([]string, len(keywords)) |
| 319 | copy(out, keywords) |
| 320 | return out |
| 321 | } |
| 322 | |
| 323 | // words gathers a group of them, which reads better at the call sites above |
| 324 | // than a slice literal does. |
| 325 | func words(list ...string) []string { return list } |
| 326 | |
| 327 | // classify pairs every word in a group with the class it belongs to. |
| 328 | func classify(class syntax.Class, list []string) map[string]syntax.Class { |
| 329 | out := make(map[string]syntax.Class, len(list)) |
| 330 | for _, word := range list { |
| 331 | out[word] = class |
| 332 | } |
| 333 | return out |
| 334 | } |
| 335 | |
| 336 | // merge folds the groups into one table. An earlier group wins a word a later |
| 337 | // one repeats, which is what keeps a keyword a keyword. |
| 338 | func merge(groups ...map[string]syntax.Class) map[string]syntax.Class { |
| 339 | out := map[string]syntax.Class{} |
| 340 | for _, group := range groups { |
| 341 | for word, class := range group { |
| 342 | if _, taken := out[word]; !taken { |
| 343 | out[word] = class |
| 344 | } |
| 345 | } |
| 346 | } |
| 347 | return out |
| 348 | } |