| 📦 Turbo Go 3d7798b k33g 9h ago | 1 | package golang |
| 2 | |
| 3 | import ( |
| 4 | "go/scanner" |
| 5 | "go/token" |
| 6 | |
| 7 | "rickub.com/turbo-editors/turbo-core/syntax" |
| 8 | ) |
| 9 | |
| 10 | // rawToken is one token as go/scanner reports it, reduced to what colouring |
| 11 | // needs: a byte range and the class it belongs to. |
| 12 | type rawToken struct { |
| 13 | start int |
| 14 | end int |
| 15 | tok token.Token |
| 16 | lit string |
| 17 | } |
| 18 | |
| 19 | // scanTokens tokenises src, ignoring every syntax error. |
| 20 | // |
| 21 | // Errors are expected: the file is being typed into. The scanner still returns |
| 22 | // a usable token for broken input — an unterminated string comes back as a |
| 23 | // STRING running to the end of the line — which is exactly what keeps the |
| 24 | // colours steady while the user types. |
| 25 | func scanTokens(src string) []rawToken { |
| 26 | fileSet := token.NewFileSet() |
| 27 | file := fileSet.AddFile("", fileSet.Base(), len(src)) |
| 28 | |
| 29 | var s scanner.Scanner |
| 30 | s.Init(file, []byte(src), func(token.Position, string) {}, scanner.ScanComments) |
| 31 | |
| 32 | var tokens []rawToken |
| 33 | for { |
| 34 | pos, tok, lit := s.Scan() |
| 35 | if tok == token.EOF { |
| 36 | return tokens |
| 37 | } |
| 38 | |
| 39 | start := file.Offset(pos) |
| 40 | width := tokenWidth(tok, lit) |
| 41 | if width == 0 { |
| 42 | continue // an automatically inserted semicolon covers no text |
| 43 | } |
| 44 | tokens = append(tokens, rawToken{start: start, end: start + width, tok: tok, lit: lit}) |
| 45 | } |
| 46 | } |
| 47 | |
| 48 | // tokenWidth returns how many bytes of source a token covers. |
| 49 | // |
| 50 | // Operators and punctuation come back with an empty literal, so their width is |
| 51 | // that of their spelling; a semicolon the scanner inserted itself covers |
| 52 | // nothing at all. |
| 53 | func tokenWidth(tok token.Token, lit string) int { |
| 54 | if tok == token.SEMICOLON && lit == "\n" { |
| 55 | return 0 |
| 56 | } |
| 57 | if lit != "" { |
| 58 | return len(lit) |
| 59 | } |
| 60 | return len(tok.String()) |
| 61 | } |
| 62 | |
| 63 | // classify assigns a colouring class to each token, using the token that |
| 64 | // follows and the one before where that is what distinguishes them: an |
| 65 | // identifier before "(" is a call, and one after "type" is a type name. |
| 66 | func classify(tokens []rawToken) []syntax.Class { |
| 67 | classes := make([]syntax.Class, len(tokens)) |
| 68 | for i, t := range tokens { |
| 69 | classes[i] = classOf(t, previous(tokens, i), next(tokens, i)) |
| 70 | } |
| 71 | return classes |
| 72 | } |
| 73 | |
| 74 | // literalClasses are the tokens whose class follows from the token alone. |
| 75 | var literalClasses = map[token.Token]syntax.Class{ |
| 76 | token.COMMENT: syntax.ClassComment, |
| 77 | token.STRING: syntax.ClassString, |
| 78 | token.CHAR: syntax.ClassChar, |
| 79 | token.INT: syntax.ClassNumber, |
| 80 | token.FLOAT: syntax.ClassNumber, |
| 81 | token.IMAG: syntax.ClassNumber, |
| 82 | } |
| 83 | |
| 84 | // classOf returns the class of a single token, given its neighbours. |
| 85 | func classOf(t rawToken, before, after token.Token) syntax.Class { |
| 86 | if class, ok := literalClasses[t.tok]; ok { |
| 87 | return class |
| 88 | } |
| 89 | if t.tok == token.IDENT { |
| 90 | return identifierClass(t.lit, before, after) |
| 91 | } |
| 92 | return symbolClass(t.tok) |
| 93 | } |
| 94 | |
| 95 | // symbolClass returns the class of anything that is neither a literal nor an |
| 96 | // identifier: a keyword, a bracket, or an operator. |
| 97 | func symbolClass(tok token.Token) syntax.Class { |
| 98 | switch { |
| 99 | case tok.IsKeyword(): |
| 100 | return syntax.ClassKeyword |
| 101 | case isPunctuation(tok): |
| 102 | return syntax.ClassPunctuation |
| 103 | case tok.IsOperator(): |
| 104 | return syntax.ClassOperator |
| 105 | } |
| 106 | return syntax.ClassIdentifier |
| 107 | } |
| 108 | |
| 109 | // predeclaredClasses are the identifiers the language itself provides, and |
| 110 | // what each of them is. They are recognised by name because they are not |
| 111 | // keywords: a file may shadow "len" or "any", and colouring it as the |
| 112 | // predeclared one anyway is what every other Go editor does too. |
| 113 | var predeclaredClasses = map[string]syntax.Class{ |
| 114 | // Types. |
| 115 | "any": syntax.ClassType, "bool": syntax.ClassType, "byte": syntax.ClassType, |
| 116 | "comparable": syntax.ClassType, "complex64": syntax.ClassType, "complex128": syntax.ClassType, |
| 117 | "error": syntax.ClassType, "float32": syntax.ClassType, "float64": syntax.ClassType, |
| 118 | "int": syntax.ClassType, "int8": syntax.ClassType, "int16": syntax.ClassType, |
| 119 | "int32": syntax.ClassType, "int64": syntax.ClassType, "rune": syntax.ClassType, |
| 120 | "string": syntax.ClassType, "uint": syntax.ClassType, "uint8": syntax.ClassType, |
| 121 | "uint16": syntax.ClassType, "uint32": syntax.ClassType, "uint64": syntax.ClassType, |
| 122 | "uintptr": syntax.ClassType, |
| 123 | |
| 124 | // Constants. |
| 125 | "true": syntax.ClassConstant, "false": syntax.ClassConstant, |
| 126 | "iota": syntax.ClassConstant, "nil": syntax.ClassConstant, |
| 127 | |
| 128 | // Functions. |
| 129 | "append": syntax.ClassBuiltin, "cap": syntax.ClassBuiltin, "clear": syntax.ClassBuiltin, |
| 130 | "close": syntax.ClassBuiltin, "complex": syntax.ClassBuiltin, "copy": syntax.ClassBuiltin, |
| 131 | "delete": syntax.ClassBuiltin, "imag": syntax.ClassBuiltin, "len": syntax.ClassBuiltin, |
| 132 | "make": syntax.ClassBuiltin, "max": syntax.ClassBuiltin, "min": syntax.ClassBuiltin, |
| 133 | "new": syntax.ClassBuiltin, "panic": syntax.ClassBuiltin, "print": syntax.ClassBuiltin, |
| 134 | "println": syntax.ClassBuiltin, "real": syntax.ClassBuiltin, "recover": syntax.ClassBuiltin, |
| 135 | } |
| 136 | |
| 137 | // identifierClass tells apart the several things an identifier can be. |
| 138 | func identifierClass(name string, before, after token.Token) syntax.Class { |
| 139 | if class, ok := predeclaredClasses[name]; ok { |
| 140 | return class |
| 141 | } |
| 142 | |
| 143 | class := syntax.ClassIdentifier |
| 144 | switch { |
| 145 | case before == token.TYPE, before == token.STRUCT, before == token.INTERFACE: |
| 146 | class = syntax.ClassType |
| 147 | case after == token.LPAREN, before == token.FUNC: |
| 148 | class = syntax.ClassFunction |
| 149 | } |
| 150 | return class |
| 151 | } |
| 152 | |
| 153 | // previous returns the token before index i, or ILLEGAL at the start. |
| 154 | func previous(tokens []rawToken, i int) token.Token { |
| 155 | if i == 0 { |
| 156 | return token.ILLEGAL |
| 157 | } |
| 158 | return tokens[i-1].tok |
| 159 | } |
| 160 | |
| 161 | // next returns the token after index i, or ILLEGAL at the end. |
| 162 | func next(tokens []rawToken, i int) token.Token { |
| 163 | if i+1 >= len(tokens) { |
| 164 | return token.ILLEGAL |
| 165 | } |
| 166 | return tokens[i+1].tok |
| 167 | } |
| 168 | |
| 169 | // isPunctuation reports whether a token is structure rather than computation. |
| 170 | // Brackets, commas and the like are usually themed more quietly than the |
| 171 | // operators that actually do something. |
| 172 | func isPunctuation(tok token.Token) bool { |
| 173 | switch tok { |
| 174 | case token.LPAREN, token.RPAREN, |
| 175 | token.LBRACK, token.RBRACK, |
| 176 | token.LBRACE, token.RBRACE, |
| 177 | token.COMMA, token.SEMICOLON, token.COLON, token.PERIOD: |
| 178 | return true |
| 179 | default: |
| 180 | return false |
| 181 | } |
| 182 | } |