package golang import ( "go/scanner" "go/token" "rickub.com/turbo-editors/turbo-core/syntax" ) // rawToken is one token as go/scanner reports it, reduced to what colouring // needs: a byte range and the class it belongs to. type rawToken struct { start int end int tok token.Token lit string } // scanTokens tokenises src, ignoring every syntax error. // // Errors are expected: the file is being typed into. The scanner still returns // a usable token for broken input — an unterminated string comes back as a // STRING running to the end of the line — which is exactly what keeps the // colours steady while the user types. func scanTokens(src string) []rawToken { fileSet := token.NewFileSet() file := fileSet.AddFile("", fileSet.Base(), len(src)) var s scanner.Scanner s.Init(file, []byte(src), func(token.Position, string) {}, scanner.ScanComments) var tokens []rawToken for { pos, tok, lit := s.Scan() if tok == token.EOF { return tokens } start := file.Offset(pos) width := tokenWidth(tok, lit) if width == 0 { continue // an automatically inserted semicolon covers no text } tokens = append(tokens, rawToken{start: start, end: start + width, tok: tok, lit: lit}) } } // tokenWidth returns how many bytes of source a token covers. // // Operators and punctuation come back with an empty literal, so their width is // that of their spelling; a semicolon the scanner inserted itself covers // nothing at all. func tokenWidth(tok token.Token, lit string) int { if tok == token.SEMICOLON && lit == "\n" { return 0 } if lit != "" { return len(lit) } return len(tok.String()) } // classify assigns a colouring class to each token, using the token that // follows and the one before where that is what distinguishes them: an // identifier before "(" is a call, and one after "type" is a type name. func classify(tokens []rawToken) []syntax.Class { classes := make([]syntax.Class, len(tokens)) for i, t := range tokens { classes[i] = classOf(t, previous(tokens, i), next(tokens, i)) } return classes } // literalClasses are the tokens whose class follows from the token alone. var literalClasses = map[token.Token]syntax.Class{ token.COMMENT: syntax.ClassComment, token.STRING: syntax.ClassString, token.CHAR: syntax.ClassChar, token.INT: syntax.ClassNumber, token.FLOAT: syntax.ClassNumber, token.IMAG: syntax.ClassNumber, } // classOf returns the class of a single token, given its neighbours. func classOf(t rawToken, before, after token.Token) syntax.Class { if class, ok := literalClasses[t.tok]; ok { return class } if t.tok == token.IDENT { return identifierClass(t.lit, before, after) } return symbolClass(t.tok) } // symbolClass returns the class of anything that is neither a literal nor an // identifier: a keyword, a bracket, or an operator. func symbolClass(tok token.Token) syntax.Class { switch { case tok.IsKeyword(): return syntax.ClassKeyword case isPunctuation(tok): return syntax.ClassPunctuation case tok.IsOperator(): return syntax.ClassOperator } return syntax.ClassIdentifier } // predeclaredClasses are the identifiers the language itself provides, and // what each of them is. They are recognised by name because they are not // keywords: a file may shadow "len" or "any", and colouring it as the // predeclared one anyway is what every other Go editor does too. var predeclaredClasses = map[string]syntax.Class{ // Types. "any": syntax.ClassType, "bool": syntax.ClassType, "byte": syntax.ClassType, "comparable": syntax.ClassType, "complex64": syntax.ClassType, "complex128": syntax.ClassType, "error": syntax.ClassType, "float32": syntax.ClassType, "float64": syntax.ClassType, "int": syntax.ClassType, "int8": syntax.ClassType, "int16": syntax.ClassType, "int32": syntax.ClassType, "int64": syntax.ClassType, "rune": syntax.ClassType, "string": syntax.ClassType, "uint": syntax.ClassType, "uint8": syntax.ClassType, "uint16": syntax.ClassType, "uint32": syntax.ClassType, "uint64": syntax.ClassType, "uintptr": syntax.ClassType, // Constants. "true": syntax.ClassConstant, "false": syntax.ClassConstant, "iota": syntax.ClassConstant, "nil": syntax.ClassConstant, // Functions. "append": syntax.ClassBuiltin, "cap": syntax.ClassBuiltin, "clear": syntax.ClassBuiltin, "close": syntax.ClassBuiltin, "complex": syntax.ClassBuiltin, "copy": syntax.ClassBuiltin, "delete": syntax.ClassBuiltin, "imag": syntax.ClassBuiltin, "len": syntax.ClassBuiltin, "make": syntax.ClassBuiltin, "max": syntax.ClassBuiltin, "min": syntax.ClassBuiltin, "new": syntax.ClassBuiltin, "panic": syntax.ClassBuiltin, "print": syntax.ClassBuiltin, "println": syntax.ClassBuiltin, "real": syntax.ClassBuiltin, "recover": syntax.ClassBuiltin, } // identifierClass tells apart the several things an identifier can be. func identifierClass(name string, before, after token.Token) syntax.Class { if class, ok := predeclaredClasses[name]; ok { return class } class := syntax.ClassIdentifier switch { case before == token.TYPE, before == token.STRUCT, before == token.INTERFACE: class = syntax.ClassType case after == token.LPAREN, before == token.FUNC: class = syntax.ClassFunction } return class } // previous returns the token before index i, or ILLEGAL at the start. func previous(tokens []rawToken, i int) token.Token { if i == 0 { return token.ILLEGAL } return tokens[i-1].tok } // next returns the token after index i, or ILLEGAL at the end. func next(tokens []rawToken, i int) token.Token { if i+1 >= len(tokens) { return token.ILLEGAL } return tokens[i+1].tok } // isPunctuation reports whether a token is structure rather than computation. // Brackets, commas and the like are usually themed more quietly than the // operators that actually do something. func isPunctuation(tok token.Token) bool { switch tok { case token.LPAREN, token.RPAREN, token.LBRACK, token.RBRACK, token.LBRACE, token.RBRACE, token.COMMA, token.SEMICOLON, token.COLON, token.PERIOD: return true default: return false } }