| 📦 Turbo Python 6fc62ea k33g 8h ago | 1 | package pythonlang |
| 2 | |
| 3 | import ( |
| 4 | "strings" |
| 5 | "testing" |
| 6 | |
| 7 | "rickub.com/turbo-editors/turbo-core/syntax" |
| 8 | ) |
| 9 | |
| 10 | // classAt returns the class covering a rune column on a line, and whether any |
| 11 | // span covers it at all. It is how nearly every test below asks its question. |
| 12 | func classAt(spans [][]syntax.Span, line, col int) (syntax.Class, bool) { |
| 13 | if line < 0 || line >= len(spans) { |
| 14 | return 0, false |
| 15 | } |
| 16 | for _, s := range spans[line] { |
| 17 | if col >= s.Start && col < s.End { |
| 18 | return s.Class, true |
| 19 | } |
| 20 | } |
| 21 | return 0, false |
| 22 | } |
| 23 | |
| 24 | // classOfFirst returns the class of the first occurrence of word in src. |
| 25 | func classOfFirst(t *testing.T, src, word string) syntax.Class { |
| 26 | t.Helper() |
| 27 | |
| 28 | index := strings.Index(src, word) |
| 29 | if index < 0 { |
| 30 | t.Fatalf("%q does not appear in the source", word) |
| 31 | } |
| 32 | line := strings.Count(src[:index], "\n") |
| 33 | col := index - (strings.LastIndex(src[:index], "\n") + 1) |
| 34 | |
| 35 | class, ok := classAt(Highlight(src), line, col) |
| 36 | if !ok { |
| 37 | t.Fatalf("no span covers %q at line %d column %d", word, line, col) |
| 38 | } |
| 39 | return class |
| 40 | } |
| 41 | |
| 42 | // spanOfFirst returns the span covering the first occurrence of word, so that a |
| 43 | // test can check where a construct *ends* and not only what colour it is. |
| 44 | func spanOfFirst(t *testing.T, src, word string) syntax.Span { |
| 45 | t.Helper() |
| 46 | |
| 47 | index := strings.Index(src, word) |
| 48 | if index < 0 { |
| 49 | t.Fatalf("%q does not appear in the source", word) |
| 50 | } |
| 51 | line := strings.Count(src[:index], "\n") |
| 52 | col := index - (strings.LastIndex(src[:index], "\n") + 1) |
| 53 | |
| 54 | for _, s := range Highlight(src)[line] { |
| 55 | if col >= s.Start && col < s.End { |
| 56 | return s |
| 57 | } |
| 58 | } |
| 59 | t.Fatalf("no span covers %q at line %d column %d", word, line, col) |
| 60 | return syntax.Span{} |
| 61 | } |
| 62 | |
| 63 | // --- the three invariants the editor relies on ------------------------------ |
| 64 | |
| 65 | func TestHighlightReturnsOneEntryPerLine(t *testing.T) { |
| 66 | // The editor indexes the result by line number without checking, so a short |
| 67 | // result is an index out of range in the middle of a redraw. |
| 68 | tests := []struct { |
| 69 | name string |
| 70 | src string |
| 71 | want int |
| 72 | }{ |
| 73 | {"empty", "", 1}, |
| 74 | {"one line without a terminator", "x = 1", 1}, |
| 75 | {"one line with a terminator", "x = 1\n", 2}, |
| 76 | {"three lines", "a\nb\nc", 3}, |
| 77 | {"an unterminated triple quote", `x = """a` + "\nb\nc", 3}, |
| 78 | } |
| 79 | |
| 80 | for _, tc := range tests { |
| 81 | t.Run(tc.name, func(t *testing.T) { |
| 82 | if got := len(Highlight(tc.src)); got != tc.want { |
| 83 | t.Errorf("Highlight(%q) returned %d lines, want %d", tc.src, got, tc.want) |
| 84 | } |
| 85 | }) |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | func TestSpansAreInOrderAndDoNotOverlap(t *testing.T) { |
| 90 | // They are drawn in order, so two out of order paint over each other and |
| 91 | // nothing fails. This has happened in this family, in a scanner that |
| 92 | // emitted a quote after the name it belonged to. |
| 93 | const src = `#!/usr/bin/env python3 |
| 94 | """A module docstring |
| 95 | spanning two lines.""" |
| 96 | |
| 97 | import re |
| 98 | from dataclasses import dataclass |
| 99 | |
| 100 | MAX_SIZE = 1_000 |
| 101 | PATTERN = re.compile(r"\d+(\.\d+)?") |
| 102 | |
| 103 | |
| 104 | @dataclass(frozen=True) |
| 105 | class Measurement: |
| 106 | """One reading.""" |
| 107 | |
| 108 | name: str |
| 109 | value: float = 0.5 |
| 110 | |
| 111 | def scaled(self, factor: float = 1.5e-3) -> float: |
| 112 | return self.value * factor |
| 113 | |
| 114 | |
| 115 | def main() -> None: |
| 116 | for line in open("input.txt"): |
| 117 | match line.strip(): |
| 118 | case "": |
| 119 | continue |
| 120 | case other: |
| 121 | print(f"{other!r} -> {len(other)}", end="") |
| 122 | |
| 123 | |
| 124 | if __name__ == "__main__": |
| 125 | main() |
| 126 | ` |
| 127 | |
| 128 | for line, spans := range Highlight(src) { |
| 129 | previous := syntax.Span{End: -1} |
| 130 | for _, span := range spans { |
| 131 | switch { |
| 132 | case span.Start < previous.End: |
| 133 | t.Errorf("line %d: %v starts at %d, inside %v which ends at %d", |
| 134 | line, span.Class, span.Start, previous.Class, previous.End) |
| 135 | case span.Start >= span.End: |
| 136 | t.Errorf("line %d: %v is empty at %d", line, span.Class, span.Start) |
| 137 | } |
| 138 | previous = span |
| 139 | } |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | func TestBrokenSourceStillColours(t *testing.T) { |
| 144 | // Source under the cursor is invalid most of the time it is being typed. A |
| 145 | // scanner that gives up is a scanner that flickers off. |
| 146 | broken := []string{ |
| 147 | `x = "unterminated`, |
| 148 | `x = '''unterminated`, |
| 149 | "def f(", |
| 150 | "class", |
| 151 | "@", |
| 152 | "f'{", |
| 153 | "x = 0x", |
| 154 | " )))", |
| 155 | "\\", |
| 156 | "x = 1.2.3.4", |
| 157 | } |
| 158 | |
| 159 | for _, src := range broken { |
| 160 | t.Run(src, func(t *testing.T) { |
| 161 | spans := Highlight(src) |
| 162 | if len(spans) != 1 { |
| 163 | t.Fatalf("Highlight(%q) returned %d lines, want 1", src, len(spans)) |
| 164 | } |
| 165 | for _, span := range spans[0] { |
| 166 | if span.Start < 0 || span.End > len([]rune(src)) { |
| 167 | t.Errorf("%v runs from %d to %d, outside a line of %d runes", |
| 168 | span.Class, span.Start, span.End, len([]rune(src))) |
| 169 | } |
| 170 | } |
| 171 | }) |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | // --- one test per construct ------------------------------------------------- |
| 176 | |
| 177 | func TestEachTokenClass(t *testing.T) { |
| 178 | const src = `# a comment |
| 179 | import os |
| 180 | from typing import Iterator |
| 181 | |
| 182 | |
| 183 | class Shape: |
| 184 | def __init__(self, x: int = 0) -> None: |
| 185 | self.x = x |
| 186 | name = "world" |
| 187 | ratio = 1.5 |
| 188 | count = 42 |
| 189 | missing = None |
| 190 | ok = True |
| 191 | print(len(name)) |
| 192 | ` |
| 193 | |
| 194 | tests := []struct { |
| 195 | word string |
| 196 | want syntax.Class |
| 197 | }{ |
| 198 | {"# a comment", syntax.ClassComment}, |
| 199 | {"import", syntax.ClassKeyword}, |
| 200 | {"from", syntax.ClassKeyword}, |
| 201 | {"class", syntax.ClassKeyword}, |
| 202 | {"def", syntax.ClassKeyword}, |
| 203 | {"Shape", syntax.ClassType}, |
| 204 | {"int", syntax.ClassType}, |
| 205 | {"__init__", syntax.ClassBuiltin}, |
| 206 | {"self", syntax.ClassBuiltin}, |
| 207 | {`"world"`, syntax.ClassString}, |
| 208 | {"1.5", syntax.ClassNumber}, |
| 209 | {"42", syntax.ClassNumber}, |
| 210 | {"None", syntax.ClassConstant}, |
| 211 | {"True", syntax.ClassConstant}, |
| 212 | {"print", syntax.ClassBuiltin}, |
| 213 | {"len", syntax.ClassBuiltin}, |
| 214 | {":", syntax.ClassPunctuation}, |
| 215 | {"=", syntax.ClassOperator}, |
| 216 | } |
| 217 | |
| 218 | for _, test := range tests { |
| 219 | t.Run(test.word, func(t *testing.T) { |
| 220 | if got := classOfFirst(t, src, test.word); got != test.want { |
| 221 | t.Errorf("%q is %v, want %v", test.word, got, test.want) |
| 222 | } |
| 223 | }) |
| 224 | } |
| 225 | } |
| 226 | |
| 227 | func TestACallIsAFunction(t *testing.T) { |
| 228 | if got := classOfFirst(t, "n = compute(3)", "compute"); got != syntax.ClassFunction { |
| 229 | t.Errorf("compute is %v, want function", got) |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | func TestADeclaredFunctionNameIsAFunction(t *testing.T) { |
| 234 | if got := classOfFirst(t, "def parse(text):\n pass\n", "parse"); got != syntax.ClassFunction { |
| 235 | t.Errorf("parse is %v, want function", got) |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | // A class is called exactly the way a function is, so the parenthesis cannot |
| 240 | // tell them apart and the naming convention has to. |
| 241 | func TestACapitalisedNameIsATypeEvenWhenItIsCalled(t *testing.T) { |
| 242 | tests := []struct{ src, word string }{ |
| 243 | {`raise ValueError("nope")`, "ValueError"}, |
| 244 | {`thing = Measurement(1)`, "Measurement"}, |
| 245 | {`class Measurement:`, "Measurement"}, |
| 246 | {`def f() -> Measurement:`, "Measurement"}, |
| 247 | } |
| 248 | |
| 249 | for _, tc := range tests { |
| 250 | t.Run(tc.src, func(t *testing.T) { |
| 251 | if got := classOfFirst(t, tc.src, tc.word); got != syntax.ClassType { |
| 252 | t.Errorf("%q in %q is %v, want type", tc.word, tc.src, got) |
| 253 | } |
| 254 | }) |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | func TestAWordInCapitalsIsAConstant(t *testing.T) { |
| 259 | for _, word := range []string{"MAX_SIZE", "PI", "HTTP_PORT", "_PRIVATE", "V2"} { |
| 260 | t.Run(word, func(t *testing.T) { |
| 261 | src := word + " = 1" |
| 262 | if got := classOfFirst(t, src, word); got != syntax.ClassConstant { |
| 263 | t.Errorf("%s is %v, want constant", word, got) |
| 264 | } |
| 265 | }) |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | func TestASingleCapitalIsATypeNotAConstant(t *testing.T) { |
| 270 | // The constant rule wants at least two runes, so that a one-letter type |
| 271 | // variable — T, the name every generic in the standard library uses — is |
| 272 | // not read as a constant. |
| 273 | if got := classOfFirst(t, `T = TypeVar("T")`, "T"); got != syntax.ClassType { |
| 274 | t.Errorf("T is %v, want type", got) |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | func TestNumbersInEveryBaseAndShape(t *testing.T) { |
| 279 | numbers := []string{ |
| 280 | "42", "1_000", "0xFF", "0o17", "0b1010", "1.5", ".5", "1.", "1e10", |
| 281 | "1.5e-3", "1E+7", "3j", "0x_FF_FF", |
| 282 | } |
| 283 | |
| 284 | for _, number := range numbers { |
| 285 | t.Run(number, func(t *testing.T) { |
| 286 | src := "x = " + number + "\n" |
| 287 | span := spanOfFirst(t, src, number) |
| 288 | if span.Class != syntax.ClassNumber { |
| 289 | t.Errorf("%s is %v, want number", number, span.Class) |
| 290 | } |
| 291 | if got := span.End - span.Start; got != len([]rune(number)) { |
| 292 | t.Errorf("%s is coloured over %d runes, want %d", number, got, len([]rune(number))) |
| 293 | } |
| 294 | }) |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | // 0xE-1 is a hexadecimal literal minus one. The E is a digit here, not the e of |
| 299 | // an exponent, so the sign after it is an operator and not part of the number. |
| 300 | func TestAHexadecimalLiteralDoesNotSwallowTheSignAfterIt(t *testing.T) { |
| 301 | span := spanOfFirst(t, "x = 0xE-1", "0xE") |
| 302 | |
| 303 | if span.End != len("x = 0xE") { |
| 304 | t.Errorf("0xE is coloured to column %d, want %d — the minus was taken as an exponent's sign", |
| 305 | span.End, len("x = 0xE")) |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | // A float has at most one dot. Without that limit, `1.2.3` is one long number |
| 310 | // and the version string somebody is halfway through typing paints the line. |
| 311 | func TestANumberStopsAtItsSecondDot(t *testing.T) { |
| 312 | span := spanOfFirst(t, "x = 1.2.3", "1.2") |
| 313 | |
| 314 | if span.End != len("x = 1.2") { |
| 315 | t.Errorf("1.2 is coloured to column %d, want %d — the second dot was swallowed", |
| 316 | span.End, len("x = 1.2")) |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | func TestADottedAttributeIsNotANumber(t *testing.T) { |
| 321 | if got := classOfFirst(t, "value = thing.count", "count"); got != syntax.ClassIdentifier { |
| 322 | t.Errorf("count after a dot is %v, want identifier", got) |
| 323 | } |
| 324 | } |
| 325 | |
| 326 | // --- strings ---------------------------------------------------------------- |
| 327 | |
| 328 | func TestEveryStringPrefixOpensAString(t *testing.T) { |
| 329 | prefixes := []string{"", "r", "R", "b", "B", "u", "U", "f", "F", "rb", "br", "fr", "rf", "Rb", "BR"} |
| 330 | |
| 331 | for _, prefix := range prefixes { |
| 332 | t.Run("prefix "+prefix, func(t *testing.T) { |
| 333 | literal := prefix + `"hello"` |
| 334 | src := "x = " + literal |
| 335 | span := spanOfFirst(t, src, literal) |
| 336 | if span.Class != syntax.ClassString { |
| 337 | t.Errorf("%s is %v, want string", literal, span.Class) |
| 338 | } |
| 339 | if got := span.End - span.Start; got != len([]rune(literal)) { |
| 340 | t.Errorf("%s is coloured over %d runes, want %d — the prefix was left out", |
| 341 | literal, got, len([]rune(literal))) |
| 342 | } |
| 343 | }) |
| 344 | } |
| 345 | } |
| 346 | |
| 347 | func TestAnIdentifierEndingInAPrefixLetterIsNotAString(t *testing.T) { |
| 348 | // foo"bar" must be an identifier and a string, not one run: the prefix |
| 349 | // letters are only a prefix when nothing but them comes before the quote. |
| 350 | if got := classOfFirst(t, `foo"bar"`, "foo"); got != syntax.ClassIdentifier { |
| 351 | t.Errorf("foo before a quote is %v, want identifier", got) |
| 352 | } |
| 353 | } |
| 354 | |
| 355 | func TestATripleQuotedStringCrossesLines(t *testing.T) { |
| 356 | const src = "text = \"\"\"one\ntwo\nthree\"\"\"\nx = 1\n" |
| 357 | spans := Highlight(src) |
| 358 | |
| 359 | // Column 8 on the opening line is inside the literal; the continued lines |
| 360 | // are short, so they are asked about at their first column. |
| 361 | for _, at := range []struct{ line, col int }{{0, 8}, {1, 0}, {2, 0}} { |
| 362 | if class, ok := classAt(spans, at.line, at.col); !ok || class != syntax.ClassString { |
| 363 | t.Errorf("line %d column %d is %v (covered: %t), want string", at.line, at.col, class, ok) |
| 364 | } |
| 365 | } |
| 366 | if class, _ := classAt(spans, 3, 0); class != syntax.ClassIdentifier { |
| 367 | t.Errorf("the line after the string is %v, want identifier — the string did not close", class) |
| 368 | } |
| 369 | } |
| 370 | |
| 371 | // A lone quote inside a triple-quoted string closes nothing: it takes three. |
| 372 | // Without this the closer is the same rune the opener started with, and every |
| 373 | // docstring that quotes anything ends in the middle of itself. |
| 374 | func TestALoneQuoteInsideATripleQuotedStringClosesNothing(t *testing.T) { |
| 375 | const src = "text = \"\"\"say \"hi\" now\"\"\"\nx = 1\n" |
| 376 | |
| 377 | for _, inside := range []string{"hi", "now"} { |
| 378 | if got := classOfFirst(t, src, inside); got != syntax.ClassString { |
| 379 | t.Errorf("%q inside the docstring is %v, want string — one quote ended it", inside, got) |
| 380 | } |
| 381 | } |
| 382 | if class, _ := classAt(Highlight(src), 1, 0); class != syntax.ClassIdentifier { |
| 383 | t.Errorf("the line after it is %v, want identifier", class) |
| 384 | } |
| 385 | } |
| 386 | |
| 387 | // The same thing across a line break, which is where a docstring actually |
| 388 | // lives: the carried closer has to be three quotes, not one. |
| 389 | func TestACarriedTripleQuotedStringNeedsThreeQuotesToClose(t *testing.T) { |
| 390 | const src = "text = \"\"\"first \"quoted\"\nsecond \"also\"\nthird\"\"\"\nx = 1\n" |
| 391 | spans := Highlight(src) |
| 392 | |
| 393 | for line := range 3 { |
| 394 | if class, ok := classAt(spans, line, 7); !ok || class != syntax.ClassString { |
| 395 | t.Errorf("line %d is %v (covered: %t), want string — a lone quote closed it", line, class, ok) |
| 396 | } |
| 397 | } |
| 398 | if class, _ := classAt(spans, 3, 0); class != syntax.ClassIdentifier { |
| 399 | t.Errorf("the line after it is %v, want identifier — the string never closed", class) |
| 400 | } |
| 401 | } |
| 402 | |
| 403 | func TestOneKindOfTripleQuoteDoesNotCloseTheOther(t *testing.T) { |
| 404 | const src = "text = '''one \"\"\" two'''\nx = 1\n" |
| 405 | |
| 406 | if got := classOfFirst(t, src, `"""`); got != syntax.ClassString { |
| 407 | t.Errorf(`the """ inside a ''' string is %v, want string`, got) |
| 408 | } |
| 409 | if class, _ := classAt(Highlight(src), 1, 0); class != syntax.ClassIdentifier { |
| 410 | t.Errorf("the next line is %v, want identifier — the ''' never closed", class) |
| 411 | } |
| 412 | } |
| 413 | |
| 414 | // A single-quoted string is not allowed to cross a line break, so one that |
| 415 | // reaches the end of a line without a backslash is coloured to there and |
| 416 | // dropped. Carrying it would paint the rest of the file as a string. |
| 417 | func TestAnUnterminatedSingleQuotedStringDoesNotCrossTheLineBreak(t *testing.T) { |
| 418 | const src = "x = \"unterminated\ny = 1\n" |
| 419 | |
| 420 | if class, _ := classAt(Highlight(src), 1, 0); class != syntax.ClassIdentifier { |
| 421 | t.Errorf("the line after an unterminated string is %v, want identifier", class) |
| 422 | } |
| 423 | } |
| 424 | |
| 425 | // …but a backslash at the end of the line escapes the newline, and then it |
| 426 | // really does carry on. That is the one case where carrying is right. |
| 427 | func TestABackslashAtTheEndOfALineContinuesASingleQuotedString(t *testing.T) { |
| 428 | const src = "x = \"one\\\ntwo\"\ny = 1\n" |
| 429 | |
| 430 | if class, ok := classAt(Highlight(src), 1, 0); !ok || class != syntax.ClassString { |
| 431 | t.Errorf("the continued line is %v (covered: %t), want string", class, ok) |
| 432 | } |
| 433 | if class, _ := classAt(Highlight(src), 2, 0); class != syntax.ClassIdentifier { |
| 434 | t.Errorf("the line after the close is %v, want identifier", class) |
| 435 | } |
| 436 | } |
| 437 | |
| 438 | // In a raw string the backslash is kept in the value, but it still stops the |
| 439 | // quote after it from ending the literal — which is why rawness is not carried. |
| 440 | func TestABackslashEscapesTheQuoteInARawStringToo(t *testing.T) { |
| 441 | const src = `p = r"\"" + "after"` |
| 442 | |
| 443 | if got := classOfFirst(t, src, `"after"`); got != syntax.ClassString { |
| 444 | t.Errorf(`"after" is %v, want string — r"\"" ended one quote too early`, got) |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | func TestAHashInsideAStringIsNotAComment(t *testing.T) { |
| 449 | if got := classOfFirst(t, `url = "http://x/#anchor"`, "#anchor"); got != syntax.ClassString { |
| 450 | t.Errorf("the # inside a string is %v, want string", got) |
| 451 | } |
| 452 | } |
| 453 | |
| 454 | // --- decorators ------------------------------------------------------------- |
| 455 | |
| 456 | func TestADecoratorIsAnAttribute(t *testing.T) { |
| 457 | for _, src := range []string{"@property\n", " @property\n", "@app.route\n"} { |
| 458 | t.Run(src, func(t *testing.T) { |
| 459 | if got := classOfFirst(t, src, "@"); got != syntax.ClassAttribute { |
| 460 | t.Errorf("the decorator in %q is %v, want attribute", src, got) |
| 461 | } |
| 462 | }) |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | func TestADecoratorStopsAtItsArguments(t *testing.T) { |
| 467 | const src = `@pytest.mark.parametrize("n", [1, 2])` |
| 468 | |
| 469 | if span := spanOfFirst(t, src, "@"); span.End != len("@pytest.mark.parametrize") { |
| 470 | t.Errorf("the decorator is coloured to column %d, want %d", span.End, len("@pytest.mark.parametrize")) |
| 471 | } |
| 472 | if got := classOfFirst(t, src, `"n"`); got != syntax.ClassString { |
| 473 | t.Errorf(`the "n" argument is %v, want string`, got) |
| 474 | } |
| 475 | } |
| 476 | |
| 477 | // The same rune is the matrix-multiplication operator, and only its position |
| 478 | // tells the two apart. |
| 479 | func TestAnAtSignInTheMiddleOfALineIsAnOperator(t *testing.T) { |
| 480 | // With a space after it, the rune that follows already settles it. Without |
| 481 | // one — `a @b` is ordinary Python — position is the only thing that does, |
| 482 | // which is what this second case is for. |
| 483 | for _, src := range []string{"product = a @ b", "product = a @b"} { |
| 484 | t.Run(src, func(t *testing.T) { |
| 485 | if got := classOfFirst(t, src, "@"); got != syntax.ClassOperator { |
| 486 | t.Errorf("the @ in %q is %v, want operator", src, got) |
| 487 | } |
| 488 | }) |
| 489 | } |
| 490 | } |
| 491 | |
| 492 | // --- the soft keywords ------------------------------------------------------ |
| 493 | |
| 494 | func TestMatchAndCaseAreKeywordsWhenTheyOpenABlock(t *testing.T) { |
| 495 | const src = "match command.split():\n case [\"go\", direction]:\n pass\n" |
| 496 | |
| 497 | if got := classOfFirst(t, src, "match"); got != syntax.ClassKeyword { |
| 498 | t.Errorf("match opening a statement is %v, want keyword", got) |
| 499 | } |
| 500 | if got := classOfFirst(t, src, "case"); got != syntax.ClassKeyword { |
| 501 | t.Errorf("case opening a block is %v, want keyword", got) |
| 502 | } |
| 503 | } |
| 504 | |
| 505 | func TestMatchIsAnOrdinaryNameEverywhereElse(t *testing.T) { |
| 506 | tests := []struct { |
| 507 | name string |
| 508 | src string |
| 509 | want syntax.Class |
| 510 | }{ |
| 511 | {"assigned", "match = re.match(pattern, text)", syntax.ClassIdentifier}, |
| 512 | {"called", "if match(pattern):\n pass\n", syntax.ClassFunction}, |
| 513 | {"an argument", "use(match)", syntax.ClassIdentifier}, |
| 514 | {"annotated", "match: str = compute()", syntax.ClassIdentifier}, |
| 515 | } |
| 516 | |
| 517 | for _, tc := range tests { |
| 518 | t.Run(tc.name, func(t *testing.T) { |
| 519 | if got := classOfFirst(t, tc.src, "match"); got != tc.want { |
| 520 | t.Errorf("match in %q is %v, want %v", tc.src, got, tc.want) |
| 521 | } |
| 522 | }) |
| 523 | } |
| 524 | } |
| 525 | |
| 526 | // The boundary of the soft-keyword rule, tested rather than left to be |
| 527 | // discovered: a trailing comment hides the colon, and match reads as a name. |
| 528 | // It is the safe direction to be wrong in, and reference/languages.md says so. |
| 529 | func TestATrailingCommentHidesTheColonFromTheSoftKeywordRule(t *testing.T) { |
| 530 | if got := classOfFirst(t, "match value: # dispatch\n", "match"); got != syntax.ClassIdentifier { |
| 531 | t.Errorf("match before a trailing comment is %v; the documented limitation says identifier", got) |
| 532 | } |
| 533 | } |
| 534 | |
| 535 | // --- what the scanner deliberately does not do ------------------------------ |
| 536 | |
| 537 | // An f-string's {expression} is one flat run of string, on purpose: since |
| 538 | // Python 3.12 it may contain anything at all, and colouring it half-properly |
| 539 | // breaks a format spec like "{n:{width}}". |
| 540 | func TestAnFStringIsNotScannedAsCodeInside(t *testing.T) { |
| 541 | const src = `print(f"{count:{width}} items")` |
| 542 | |
| 543 | for _, inside := range []string{"count", "width", "items"} { |
| 544 | if got := classOfFirst(t, src, inside); got != syntax.ClassString { |
| 545 | t.Errorf("%q inside an f-string is %v; the whole literal is meant to be one string", inside, got) |
| 546 | } |
| 547 | } |
| 548 | } |
| 549 | |
| 550 | // A docstring is a string, which is what the language calls it and what help() |
| 551 | // reads back. Colouring it as a comment would be a different claim, and wrong |
| 552 | // the moment one is assigned to a name. |
| 553 | func TestADocstringIsAStringAndNotAComment(t *testing.T) { |
| 554 | const src = "def f():\n \"\"\"What it does.\"\"\"\n" |
| 555 | |
| 556 | if got := classOfFirst(t, src, `"""What it does."""`); got != syntax.ClassString { |
| 557 | t.Errorf("a docstring is %v, want string", got) |
| 558 | } |
| 559 | } |
| 560 | |
| 561 | // type is a builtin type as well as a soft keyword, and reads correctly as the |
| 562 | // type in both jobs — so it is deliberately not in isSoftKeyword. |
| 563 | func TestTypeIsTheBuiltinTypeInBothOfItsJobs(t *testing.T) { |
| 564 | for _, src := range []string{"type(value)", "type Alias = int"} { |
| 565 | if got := classOfFirst(t, src, "type"); got != syntax.ClassType { |
| 566 | t.Errorf("type in %q is %v, want type", src, got) |
| 567 | } |
| 568 | } |
| 569 | } |
| 570 | |
| 571 | // The walrus is an operator; every other colon is structure. |
| 572 | func TestTheWalrusIsAnOperatorAndAPlainColonIsNot(t *testing.T) { |
| 573 | if got := classOfFirst(t, "if (n := len(text)) > 3:\n pass\n", ":="); got != syntax.ClassOperator { |
| 574 | t.Errorf(":= is %v, want operator", got) |
| 575 | } |
| 576 | if got := classOfFirst(t, `d = {"a": 1}`, ":"); got != syntax.ClassPunctuation { |
| 577 | t.Errorf("a dict colon is %v, want punctuation", got) |
| 578 | } |
| 579 | if got := classOfFirst(t, "items[1:2]", ":"); got != syntax.ClassPunctuation { |
| 580 | t.Errorf("a slice colon is %v, want punctuation", got) |
| 581 | } |
| 582 | } |
| 583 | |
| 584 | // --- the whole thing over a real file --------------------------------------- |
| 585 | |
| 586 | func TestASweepOverRepresentativeSourceLeavesNothingUncoloured(t *testing.T) { |
| 587 | // Not every rune is coloured — whitespace is not, and neither is a rune the |
| 588 | // scanner steps over — but a *word* left with no span at all means the |
| 589 | // dispatcher fell through, which is a defect and not a decision. |
| 590 | const src = `from __future__ import annotations |
| 591 | |
| 592 | import asyncio |
| 593 | from typing import Any |
| 594 | |
| 595 | |
| 596 | async def gather(*tasks: Any, timeout: float = 1.0) -> list[Any]: |
| 597 | async with asyncio.timeout(timeout): |
| 598 | return await asyncio.gather(*tasks) |
| 599 | |
| 600 | |
| 601 | class Registry(dict[str, int]): |
| 602 | __slots__ = () |
| 603 | |
| 604 | def add(self, key: str, /, *, count: int = 1) -> None: |
| 605 | self[key] = self.get(key, 0) + count |
| 606 | |
| 607 | def __repr__(self) -> str: |
| 608 | return f"Registry({dict(self)!r})" |
| 609 | |
| 610 | |
| 611 | lambda_ = lambda x: x if x else -x |
| 612 | numbers = [n**2 for n in range(10) if n % 2 == 0] |
| 613 | mapping = {k: v for k, v in zip("abc", [1, 2, 3])} |
| 614 | ` |
| 615 | |
| 616 | spans := Highlight(src) |
| 617 | for line, text := range strings.Split(src, "\n") { |
| 618 | for col, r := range []rune(text) { |
| 619 | if !syntax.IsLetter(r) && !syntax.IsDigit(r) { |
| 620 | continue |
| 621 | } |
| 622 | if _, ok := classAt(spans, line, col); !ok { |
| 623 | t.Errorf("line %d column %d (%q) is covered by no span: %q", line, col, r, text) |
| 624 | } |
| 625 | } |
| 626 | } |
| 627 | } |