package gololang // Numbers and words: what a run of digits or letters turns out to be. import ( "strings" "unicode" "rickub.com/turbo-editors/turbo-core/syntax" ) // --- numbers ---------------------------------------------------------------- // takeNumber colours a numeric literal the way lexer.go's readNumber reads // one: digits, then a point and more digits if a digit follows the point, then // an exponent with an optional sign, then the L that makes a long or the F or // f that makes a float. // // The point is only part of the number when a digit follows it. That is what // keeps 1..3 a number and a range rather than the double 1. and a stray .3, // and it is the lexer's own test β€” `l.ch == '.' && isDigit(l.peekChar())`. // // There is no hexadecimal, no binary, no octal and no digit separator, because // the lexer has none: 0xFF is the number 0 followed by the name xFF, and 1_000 // is 1 followed by the name _000. Colouring either as one number would be // inventing a literal the interpreter will reject. // // The whole literal is one span, so every step here advances the scanner // without colouring and the single Emit at the end covers what they consumed. func takeNumber(s *syntax.LineScanner) { start := s.Pos() advanceWhile(s, syntax.IsDigit) if s.Peek(0) == '.' && syntax.IsDigit(s.Peek(1)) { s.Advance(1) advanceWhile(s, syntax.IsDigit) } takeExponent(s) takeNumberSuffix(s) s.Emit(start, s.Pos(), syntax.ClassNumber) } // takeExponent consumes e or E, an optional sign, and the digits after them. // // The digits may be none. The lexer reads 1e as a float and leaves the parser // to complain, and this scanner colours what the lexer reads. func takeExponent(s *syntax.LineScanner) { if s.Peek(0) != 'e' && s.Peek(0) != 'E' { return } s.Advance(1) if s.Peek(0) == '+' || s.Peek(0) == '-' { s.Advance(1) } advanceWhile(s, syntax.IsDigit) } // takeNumberSuffix consumes the L of a long and the F or f of a float, in // that order, when the literal ends in them. 42L is a long; 3.14F and 2.0f are // floats; the lexer accepts both cases of the F and only the upper case of // the L. func takeNumberSuffix(s *syntax.LineScanner) { if s.Peek(0) == 'L' { s.Advance(1) } if s.Peek(0) == 'F' || s.Peek(0) == 'f' { s.Advance(1) } } // advanceWhile steps over runes that match, without colouring any of them. It // is what a construct emitted as a single span uses in place of TakeWhile, // which would colour each run it consumed and leave the Emit overlapping it. func advanceWhile(s *syntax.LineScanner, matches func(rune) bool) { for !s.AtEnd() && matches(s.Peek(0)) { s.Advance(1) } } // --- words ------------------------------------------------------------------ // isIdentifierStart reports whether a rune may begin a name, by the lexer's // own isLetter: any Unicode letter or mark, an underscore, or an emoji. // // This is wider than turbo-core's ASCII IsLetter on purpose. Golo lets you // write `let πŸ˜€ = 1` and `function πŸš€launch = { … }`, and a scanner that left // those uncoloured would be telling a reader they are not names when the // interpreter says they are. func isIdentifierStart(r rune) bool { return unicode.IsLetter(r) || unicode.IsMark(r) || r == '_' || isEmoji(r) } // isWordRune reports whether a rune may continue a name: whatever may begin // one, or a digit. func isWordRune(r rune) bool { return isIdentifierStart(r) || syntax.IsDigit(r) } // emojiBlocks are the four Unicode blocks the lexer admits into a name, each // as its first and last rune. They are the lexer's own, copied rather than // widened: the general symbol blocks are left out there because they hold the // operators. var emojiBlocks = [][2]rune{ {0x1F600, 0x1F64F}, // emoticons {0x1F300, 0x1F5FF}, // miscellaneous symbols and pictographs {0x1F680, 0x1F6FF}, // transport and map symbols {0x1F900, 0x1F9FF}, // supplemental symbols and pictographs } // isEmoji reports whether a rune is in one of the blocks the lexer admits into // a name. func isEmoji(r rune) bool { for _, block := range emojiBlocks { if r >= block[0] && r <= block[1] { return true } } return false } // takeWord colours a name, deciding what kind of thing it is from the word // itself and from the rune that follows it β€” and, for two keywords, colours // the name that follows *them*, because that name means something only there. func takeWord(s *syntax.LineScanner) { start := s.Pos() advanceWhile(s, isWordRune) word := wordAt(s, start) s.Emit(start, s.Pos(), classOfWord(word, s.Peek(0))) switch word { case "module", "import": takeModulePath(s) case "function": takeDeclaredFunctionName(s) } } // takeModulePath colours the dotted name after module or import as one span: // hello.World, gololang.Errors, java.util.List. // // It is one span because it is one name β€” a module has no parts a program can // take apart β€” and ClassType is the nearest of the seventeen classes: a module // path names a thing rather than holding a value, and the reading it has to be // saved from is the one where gololang.Errors looks like a variable called // gololang with something done to it. // // A dot is only taken when a name follows it, so `import a.` at the end of a // half-typed line stops before the dot and leaves it as punctuation. func takeModulePath(s *syntax.LineScanner) { s.SkipSpaces() if !isIdentifierStart(s.Peek(0)) { return } start := s.Pos() advanceWhile(s, isWordRune) for s.Peek(0) == '.' && isIdentifierStart(s.Peek(1)) { s.Advance(1) advanceWhile(s, isWordRune) } s.Emit(start, s.Pos(), syntax.ClassType) } // takeDeclaredFunctionName colours the name after the function keyword as a // function. // // Everywhere else a name is a function because a parenthesis follows it, and // a declaration is the one place that is not true: `function main = |args|` // has the name followed by an equals sign. Without this, every function a // file declares would be coloured as an ordinary variable at the one place a // reader looks for it. func takeDeclaredFunctionName(s *syntax.LineScanner) { s.SkipSpaces() if !isIdentifierStart(s.Peek(0)) { return } start := s.Pos() advanceWhile(s, isWordRune) s.Emit(start, s.Pos(), syntax.ClassFunction) } // wordAt returns the word running from start to the scanner's position. func wordAt(s *syntax.LineScanner, start int) string { var b strings.Builder for at := start; at < s.Pos(); at++ { b.WriteRune(s.Peek(at - s.Pos())) } return b.String() } // classOfWord decides what a word is, given the rune that follows it. // // The order is the design. A word the language names is what the language // says it is: a keyword, one of the three literal constants, or one of the // interpreter's built-in functions. After that comes the case rule, which in // Golo is a convention rather than a lexical fact: structs, unions and their // variants are capitalised by everybody β€” Point, Shape, Circle, Some, None β€” // and nothing else customarily is, so a capitalised word is coloured as a // type. A lower-case word followed by a parenthesis is a call, and anything // else is a name. // // What the case rule costs is that a variant's constructor is coloured as a // type β€” Circle(1.0) and Result_Failure("no") look like types applied to // arguments β€” and a capitalised variable, which Golo permits, is coloured as // one too. Nothing in the syntax separates them, and inventing a separation // would mean being wrong in both directions instead of one. func classOfWord(word string, next rune) syntax.Class { if class, known := knownWords[word]; known { return class } if startsUpperCase(word) { return syntax.ClassType } if next == '(' { return syntax.ClassFunction } return syntax.ClassIdentifier } // startsUpperCase reports whether a word begins with an ASCII capital, which // is what the convention means by a type's name. func startsUpperCase(word string) bool { return word != "" && word[0] >= 'A' && word[0] <= 'Z' } // knownWords is every word the language itself names, and what each one is. // // It is one table rather than three because it answers one question. The three // groups below are kept apart only so that each can carry the reasoning that // belongs to it. var knownWords = merge( classify(syntax.ClassKeyword, keywords), classify(syntax.ClassConstant, constants), classify(syntax.ClassBuiltin, builtinFunctions), ) // keywords are the words Golo reserves, taken from token/token.go's keyword // table in GoloScript β€” every entry of it except the three literal values, // which are constants below. // // The word operators are here as keywords: and, or, not, is, isnt, oftype and // orIfNull are reserved words that happen to compute something, and a reader // meets them as words. There is no `then`-less if or `elseif`; `else if` is // two keywords. var keywords = words( "and", "augment", "augmentation", "await", "break", "case", "catch", "continue", "else", "finally", "for", "foreach", "function", "if", "import", "in", "is", "isnt", "let", "local", "match", "module", "not", "oftype", "or", "orIfNull", "otherwise", "return", "spawn", "struct", "then", "throw", "try", "union", "var", "when", "while", "with", ) // constants are the values a reader meets as the language's own. They are // keywords to the lexer and values to the reader, and every editor in this // family colours them as constants. var constants = words("true", "false", "null") // builtinFunctions are the functions the interpreter provides without an // import, read out of evaluator.BuiltinNames() in GoloScript rather than // remembered β€” 162 names, less the five prefixed with a double underscore, // which are the test runner's own counters and which the language server // likewise keeps out of its completion list. // // Some, None, Ok and Err are deliberately not here. In Golo they are not built // in: they are the variants of ordinary unions declared in gololang.Errors, // available only after `import gololang.Errors`, so they take the colour every // other capitalised name does. DynamicObject *is* here, capital and all, // because it is a builtin function that happens to be spelt like a type. var builtinFunctions = words( "DynamicObject", "abs", "appendFile", "array", "chanClose", "chanReceive", "chanSend", "channel", "currentAbsDir", "currentDir", "currentTime", "currentTimeMillis", "currentTimeNano", "dateString", "dateTimeString", "deleteFile", "escapeJSON", "execCombinedOutput", "execCommand", "fileExists", "fileInfo", "float", "formatTime", "fromJSON", "getenv", "head", "httpDelete", "httpDeleteStream", "httpGet", "httpGetStream", "httpPost", "httpPostStream", "httpPut", "httpPutStream", "httpServe", "httpStop", "information", "int", "isNotNull", "isNull", "len", "length", "list", "listDir", "map", "mcpAddResource", "mcpAddTool", "mcpCallTool", "mcpConnectHTTP", "mcpConnectStdio", "mcpCreateServer", "mcpDisconnect", "mcpListResources", "mcpListTools", "mcpReadResource", "mcpRunHTTP", "mcpRunStdio", "mcpStopServer", "mkDir", "mutex", "mutexLock", "mutexUnlock", "now", "observable", "observableFilter", "observableGet", "observableMap", "observableOnChange", "observableSet", "openAIChatCompletion", "openAIChatCompletionStream", "openAICreateEmbedding", "openAINewClient", "parseTime", "pow", "print", "println", "push", "raise", "range", "read", "readFile", "readln", "require", "requireNotNull", "set", "setenv", "sharedGet", "sharedSet", "sharedState", "sharedUpdate", "sleep", "sqrt", "str", "tail", "template", "timeString", "toJSON", "tuiClick", "tuiComponentIds", "tuiDisplayWidth", "tuiEmit", "tuiFocus", "tuiFrame", "tuiGetProp", "tuiHasComponent", "tuiHide", "tuiIsFocused", "tuiIsVisible", "tuiLoad", "tuiLoadStyle", "tuiNew", "tuiOff", "tuiOn", "tuiQuit", "tuiRenderMarkdown", "tuiRun", "tuiSetProp", "tuiShow", "tuiSize", "tuiWheel", "tuiZoneOf", "tupleFromArray", "type", "uiConfirm", "uiError", "uiGetColorCode", "uiInfo", "uiMarkdownRender", "uiMarkdownStreamAppend", "uiMarkdownStreamEnd", "uiMarkdownStreamStart", "uiPrint", "uiPrintMultiStyle", "uiPrintln", "uiPrompt", "uiPromptMultiline", "uiPromptPassword", "uiSpinnerError", "uiSpinnerNew", "uiSpinnerSetFrames", "uiSpinnerSetPrefix", "uiSpinnerSetSuffix", "uiSpinnerStart", "uiSpinnerStop", "uiSpinnerSuccess", "uiSuccess", "uiWarning", "vector", "wasmCallNumbers", "wasmCallString", "wasmClose", "wasmHasFunction", "wasmLoad", "wasmRegisterStringHandler", "wasmShutdown", "writeFile", ) // Builtins returns the interpreter's built-in function names, sorted, so a // test can hold the table above to what a real golo answers. func Builtins() []string { out := make([]string, len(builtinFunctions)) copy(out, builtinFunctions) return out } // Keywords returns the reserved words the scanner colours as keywords, so a // test can hold the table above to what a real golo reserves. func Keywords() []string { out := make([]string, len(keywords)) copy(out, keywords) return out } // words gathers a group of them, which reads better at the call sites above // than a slice literal does. func words(list ...string) []string { return list } // classify pairs every word in a group with the class it belongs to. func classify(class syntax.Class, list []string) map[string]syntax.Class { out := make(map[string]syntax.Class, len(list)) for _, word := range list { out[word] = class } return out } // merge folds the groups into one table. An earlier group wins a word a later // one repeats, which is what keeps a keyword a keyword. func merge(groups ...map[string]syntax.Class) map[string]syntax.Class { out := map[string]syntax.Class{} for _, group := range groups { for word, class := range group { if _, taken := out[word]; !taken { out[word] = class } } } return out }