1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

package pythonlang

import (
	"strings"
	"testing"

	"rickub.com/turbo-editors/turbo-core/syntax"
)

// TestTheLanguagesReferenceIsTrue holds docs/*/reference/languages.md to the
// scanner. Every row of its Python table that no other test here covers is
// checked, so a reference claim and the code cannot drift apart quietly.
//
// The last three cases document *limitations* rather than features — an
// all-capital class name read as a constant, `type` read as the builtin rather
// than as a soft keyword, an f-string's braces left inside the string. The
// reference says each of those in so many words, and these rows are what stops
// somebody "fixing" one without also fixing the sentence.
func TestTheLanguagesReferenceIsTrue(t *testing.T) {
	tests := []struct {
		src  string
		word string
		want syntax.Class
	}{
		{"assert x is not None", "assert", syntax.ClassKeyword},
		{"nonlocal counter", "nonlocal", syntax.ClassKeyword},
		{"async def f() -> None:\n    pass\n", "async", syntax.ClassKeyword},
		{"x = NotImplemented", "NotImplemented", syntax.ClassConstant},
		{"x = Ellipsis", "Ellipsis", syntax.ClassConstant},
		{"if __debug__:\n    pass\n", "__debug__", syntax.ClassConstant},
		{"x: frozenset = frozenset()", "frozenset", syntax.ClassType},
		{"x: memoryview = m", "memoryview", syntax.ClassType},
		{"x = isinstance(v, int)", "isinstance", syntax.ClassBuiltin},
		{"x = sorted(v)", "sorted", syntax.ClassBuiltin},
		{"def f(cls) -> None:\n    pass\n", "cls", syntax.ClassBuiltin},
		{"x = obj.__name__", "__name__", syntax.ClassBuiltin},
		{`x = rb"raw bytes"`, `rb"raw bytes"`, syntax.ClassString},
		{`x = BR"raw bytes"`, `BR"raw bytes"`, syntax.ClassString},
		{`x = u"text"`, `u"text"`, syntax.ClassString},
		{"x = 0o17", "0o17", syntax.ClassNumber},
		{"x = 0b1010", "0b1010", syntax.ClassNumber},
		{"x = 3j", "3j", syntax.ClassNumber},
		{"x = 1E+7", "1E+7", syntax.ClassNumber},
		{"#!/usr/bin/env python3", "#!/usr/bin/env python3", syntax.ClassComment},
		{"@property\ndef x(self):\n    pass\n", "@property", syntax.ClassAttribute},
		{"if (n := f()) > 1:\n    pass\n", ":=", syntax.ClassOperator},
		{"d = {1: 2}", ":", syntax.ClassPunctuation},
		{"x = a @ b", "@", syntax.ClassOperator},
		{"x = 1 + \\", "\\", syntax.ClassPunctuation},
		{"x = [1, 2]", "[", syntax.ClassPunctuation},

		// The documented limitations.
		{"HTTP = 1", "HTTP", syntax.ClassConstant},
		{"type Alias = int", "type", syntax.ClassType},
		{"match value:  # dispatch\n", "match", syntax.ClassIdentifier},
	}

	for _, test := range tests {
		t.Run(test.word, func(t *testing.T) {
			index := strings.Index(test.src, test.word)
			if index < 0 {
				t.Fatalf("%q not in %q", test.word, test.src)
			}
			got, ok := classAt(Highlight(test.src), 0, index)
			if !ok || got != test.want {
				t.Errorf("%q in %q is %v (covered %v), want %v", test.word, test.src, got, ok, test.want)
			}
		})
	}
}

// The reference's Python table says the class `char` is produced by nothing
// here, because Python has no character type. That is a claim about every span
// the scanner can ever emit, so it is checked over a file that uses every
// construct the table names.
func TestTheScannerNeverProducesACharacterClass(t *testing.T) {
	const src = "#!/usr/bin/env python3\n" +
		"x = 'single'\n" +
		"y = \"double\"\n" +
		"z = b'bytes'\n" +
		"w = '''triple'''\n"

	for line, spans := range Highlight(src) {
		for _, span := range spans {
			if span.Class == syntax.ClassChar {
				t.Errorf("line %d holds a char span at %d; Python has no character literal", line, span.Start)
			}
		}
	}
}