rickub
nandi/gleanpublic⑂ Fork 0
⑂ c3ee647
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Refine language detection and expand supported languagesUnverified

Julien Robert committed 2026-05-07T22:01:50+02:00 Browse files
c3ee647 parent: e130c67
modified internal/cluster/article.go +1 -1
@@ -445,7 +445,7 @@ func (e *Engine) DetectArticleLanguages(ctx context.Context) error {
445445 updated := 0
446446 for j, lang := range langs {
447447 if lang == "" {
448- lang = "en"
448+ continue
449449 }
450450 res, err := stmt.ExecContext(ctx, lang, sub[j].id)
451451 if err != nil {
@@ -445,7 +445,7 @@ func (e *Engine) DetectArticleLanguages(ctx context.Context) error {
445 updated := 0445 updated := 0
446 for j, lang := range langs {446 for j, lang := range langs {
447 if lang == "" {447 if lang == "" {
448- lang = "en"448+ continue
449 }449 }
450 res, err := stmt.ExecContext(ctx, lang, sub[j].id)450 res, err := stmt.ExecContext(ctx, lang, sub[j].id)
451 if err != nil {451 if err != nil {
modified internal/cluster/jaccard_test.go +1 -1
@@ -679,7 +679,7 @@ func seedArticleRecData(t *testing.T, ctx context.Context, dbs *db.Store) {
679679 }
680680
681681 likes := []struct{ uri, author, feed, article string }{
682- {"at://alice/like/1", "did:test:alice", "https://tech.com/feed", "https://tech.com/go"},
682+ {"at://bob/like/1", "did:test:bob", "https://tech.com/feed", "https://tech.com/go"},
683683 {"at://bob/like/2", "did:test:bob", "https://dev.com/feed", "https://dev.com/rust"},
684684 {"at://bob/like/3", "did:test:bob", "https://dev.com/feed", "https://dev.com/cook"},
685685 {"at://bob/like/4", "did:test:bob", "https://dev.com/feed", "https://dev.com/python"},
@@ -679,7 +679,7 @@ func seedArticleRecData(t *testing.T, ctx context.Context, dbs *db.Store) {
679 }679 }
680 680
681 likes := []struct{ uri, author, feed, article string }{681 likes := []struct{ uri, author, feed, article string }{
682- {"at://alice/like/1", "did:test:alice", "https://tech.com/feed", "https://tech.com/go"},682+ {"at://bob/like/1", "did:test:bob", "https://tech.com/feed", "https://tech.com/go"},
683 {"at://bob/like/2", "did:test:bob", "https://dev.com/feed", "https://dev.com/rust"},683 {"at://bob/like/2", "did:test:bob", "https://dev.com/feed", "https://dev.com/rust"},
684 {"at://bob/like/3", "did:test:bob", "https://dev.com/feed", "https://dev.com/cook"},684 {"at://bob/like/3", "did:test:bob", "https://dev.com/feed", "https://dev.com/cook"},
685 {"at://bob/like/4", "did:test:bob", "https://dev.com/feed", "https://dev.com/python"},685 {"at://bob/like/4", "did:test:bob", "https://dev.com/feed", "https://dev.com/python"},
modified internal/cluster/llm.go +3 -4
@@ -7,6 +7,8 @@ import (
77
88 "github.com/openai/openai-go"
99 "github.com/openai/openai-go/option"
10+
11+ "pkg.rbrt.fr/glean/internal/langdetect"
1012 )
1113
1214 type LLMClient struct {
@@ -59,9 +61,6 @@ func (c *LLMClient) DetectLanguages(ctx context.Context, texts []string) ([]stri
5961 content := resp.Choices[0].Message.Content
6062 lines := strings.Split(strings.TrimSpace(content), "\n")
6163 result := make([]string, len(texts))
62- for i := range result {
63- result[i] = "en"
64- }
6564 for i, line := range lines {
6665 if i >= len(result) {
6766 break
@@ -69,7 +68,7 @@ func (c *LLMClient) DetectLanguages(ctx context.Context, texts []string) ([]stri
6968 code := strings.TrimSpace(line)
7069 code = strings.TrimPrefix(code, fmt.Sprintf("%d.", i+1))
7170 code = strings.TrimSpace(code)
72- if code != "" {
71+ if langdetect.IsKnown(code) {
7372 result[i] = code
7473 }
7574 }
@@ -7,6 +7,8 @@ import (
7 7
8 "github.com/openai/openai-go"8 "github.com/openai/openai-go"
9 "github.com/openai/openai-go/option"9 "github.com/openai/openai-go/option"
10+
11+ "pkg.rbrt.fr/glean/internal/langdetect"
10 )12 )
11 13
12 type LLMClient struct {14 type LLMClient struct {
@@ -59,9 +61,6 @@ func (c *LLMClient) DetectLanguages(ctx context.Context, texts []string) ([]stri
59 content := resp.Choices[0].Message.Content61 content := resp.Choices[0].Message.Content
60 lines := strings.Split(strings.TrimSpace(content), "\n")62 lines := strings.Split(strings.TrimSpace(content), "\n")
61 result := make([]string, len(texts))63 result := make([]string, len(texts))
62- for i := range result {
63- result[i] = "en"
64- }
65 for i, line := range lines {64 for i, line := range lines {
66 if i >= len(result) {65 if i >= len(result) {
67 break66 break
@@ -69,7 +68,7 @@ func (c *LLMClient) DetectLanguages(ctx context.Context, texts []string) ([]stri
69 code := strings.TrimSpace(line)68 code := strings.TrimSpace(line)
70 code = strings.TrimPrefix(code, fmt.Sprintf("%d.", i+1))69 code = strings.TrimPrefix(code, fmt.Sprintf("%d.", i+1))
71 code = strings.TrimSpace(code)70 code = strings.TrimSpace(code)
72- if code != "" {71+ if langdetect.IsKnown(code) {
73 result[i] = code72 result[i] = code
74 }73 }
75 }74 }
modified internal/langdetect/langdetect.go +40 -5
@@ -6,17 +6,52 @@ type Language struct {
66 }
77
88 var knownLanguages = []Language{
9+ {"ar", "Arabic"},
10+ {"bn", "Bengali"},
11+ {"bg", "Bulgarian"},
12+ {"ca", "Catalan"},
13+ {"zh", "Chinese"},
14+ {"cs", "Czech"},
15+ {"da", "Danish"},
16+ {"nl", "Dutch"},
917 {"en", "English"},
18+ {"fi", "Finnish"},
1019 {"fr", "French"},
1120 {"de", "German"},
12- {"es", "Spanish"},
13- {"pt", "Portuguese"},
21+ {"el", "Greek"},
22+ {"he", "Hebrew"},
23+ {"hi", "Hindi"},
24+ {"hu", "Hungarian"},
25+ {"id", "Indonesian"},
1426 {"it", "Italian"},
15- {"ru", "Russian"},
1627 {"ja", "Japanese"},
17- {"zh", "Chinese"},
1828 {"ko", "Korean"},
19- {"ar", "Arabic"},
29+ {"ms", "Malay"},
30+ {"nb", "Norwegian"},
31+ {"fa", "Persian"},
32+ {"pl", "Polish"},
33+ {"pt", "Portuguese"},
34+ {"ro", "Romanian"},
35+ {"ru", "Russian"},
36+ {"sk", "Slovak"},
37+ {"sl", "Slovenian"},
38+ {"es", "Spanish"},
39+ {"sv", "Swedish"},
40+ {"ta", "Tamil"},
41+ {"th", "Thai"},
42+ {"tr", "Turkish"},
43+ {"uk", "Ukrainian"},
44+ {"ur", "Urdu"},
45+ {"vi", "Vietnamese"},
46+}
47+
48+func IsKnown(code string) bool {
49+ for _, l := range knownLanguages {
50+ if l.Code == code {
51+ return true
52+ }
53+ }
54+ return false
2055 }
2156
2257 func KnownLanguages() []Language {
@@ -6,17 +6,52 @@ type Language struct {
6 }6 }
7 7
8 var knownLanguages = []Language{8 var knownLanguages = []Language{
9+ {"ar", "Arabic"},
10+ {"bn", "Bengali"},
11+ {"bg", "Bulgarian"},
12+ {"ca", "Catalan"},
13+ {"zh", "Chinese"},
14+ {"cs", "Czech"},
15+ {"da", "Danish"},
16+ {"nl", "Dutch"},
9 {"en", "English"},17 {"en", "English"},
18+ {"fi", "Finnish"},
10 {"fr", "French"},19 {"fr", "French"},
11 {"de", "German"},20 {"de", "German"},
12- {"es", "Spanish"},21+ {"el", "Greek"},
13- {"pt", "Portuguese"},22+ {"he", "Hebrew"},
23+ {"hi", "Hindi"},
24+ {"hu", "Hungarian"},
25+ {"id", "Indonesian"},
14 {"it", "Italian"},26 {"it", "Italian"},
15- {"ru", "Russian"},
16 {"ja", "Japanese"},27 {"ja", "Japanese"},
17- {"zh", "Chinese"},
18 {"ko", "Korean"},28 {"ko", "Korean"},
19- {"ar", "Arabic"},29+ {"ms", "Malay"},
30+ {"nb", "Norwegian"},
31+ {"fa", "Persian"},
32+ {"pl", "Polish"},
33+ {"pt", "Portuguese"},
34+ {"ro", "Romanian"},
35+ {"ru", "Russian"},
36+ {"sk", "Slovak"},
37+ {"sl", "Slovenian"},
38+ {"es", "Spanish"},
39+ {"sv", "Swedish"},
40+ {"ta", "Tamil"},
41+ {"th", "Thai"},
42+ {"tr", "Turkish"},
43+ {"uk", "Ukrainian"},
44+ {"ur", "Urdu"},
45+ {"vi", "Vietnamese"},
46+}
47+
48+func IsKnown(code string) bool {
49+ for _, l := range knownLanguages {
50+ if l.Code == code {
51+ return true
52+ }
53+ }
54+ return false
20 }55 }
21 56
22 func KnownLanguages() []Language {57 func KnownLanguages() []Language {
modified internal/langdetect/langdetect_test.go +9 -0
@@ -18,3 +18,12 @@ func TestKnownLanguages(t *testing.T) {
1818 }
1919 assert.Assert(t, found)
2020 }
21+
22+func TestIsKnown(t *testing.T) {
23+ assert.Assert(t, IsKnown("en"))
24+ assert.Assert(t, IsKnown("ja"))
25+ assert.Assert(t, IsKnown("zh"))
26+ assert.Assert(t, !IsKnown("xx"))
27+ assert.Assert(t, !IsKnown("bamboo-based plastic"))
28+ assert.Assert(t, !IsKnown(""))
29+}
@@ -18,3 +18,12 @@ func TestKnownLanguages(t *testing.T) {
18 }18 }
19 assert.Assert(t, found)19 assert.Assert(t, found)
20 }20 }
21+
22+func TestIsKnown(t *testing.T) {
23+ assert.Assert(t, IsKnown("en"))
24+ assert.Assert(t, IsKnown("ja"))
25+ assert.Assert(t, IsKnown("zh"))
26+ assert.Assert(t, !IsKnown("xx"))
27+ assert.Assert(t, !IsKnown("bamboo-based plastic"))
28+ assert.Assert(t, !IsKnown(""))
29+}