rickub
nandi/gleanpublic⑂ Fork 0
⑂ 6ba2d25
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Simplify packages namingUnverified

Julien Robert committed 2026-05-15T21:16:18+02:00 Browse files
6ba2d25 parent: 7ede10b
modified internal/cluster/article.go +2 -2
@@ -9,7 +9,7 @@ import (
99
1010 vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
1111
12- "pkg.rbrt.fr/glean/internal/ai"
12+ "pkg.rbrt.fr/glean/internal/ml"
1313 )
1414
1515 // embedBatchSize caps how many texts are sent in a single embedding API call.
@@ -196,7 +196,7 @@ func (e *Engine) populateContentBoost(ctx context.Context, conn *sql.Conn, userD
196196 return nil
197197 }
198198
199- queryBlob, err := ai.AvgEmbeddings(blobs, dim)
199+ queryBlob, err := ml.AvgEmbeddings(blobs, dim)
200200 if err != nil {
201201 return fmt.Errorf("serialize query vector: %w", err)
202202 }
@@ -9,7 +9,7 @@ import (
9 9
10 vec "github.com/asg017/sqlite-vec-go-bindings/cgo"10 vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
11 11
12- "pkg.rbrt.fr/glean/internal/ai"12+ "pkg.rbrt.fr/glean/internal/ml"
13 )13 )
14 14
15 // embedBatchSize caps how many texts are sent in a single embedding API call.15 // embedBatchSize caps how many texts are sent in a single embedding API call.
@@ -196,7 +196,7 @@ func (e *Engine) populateContentBoost(ctx context.Context, conn *sql.Conn, userD
196 return nil196 return nil
197 }197 }
198 198
199- queryBlob, err := ai.AvgEmbeddings(blobs, dim)199+ queryBlob, err := ml.AvgEmbeddings(blobs, dim)
200 if err != nil {200 if err != nil {
201 return fmt.Errorf("serialize query vector: %w", err)201 return fmt.Errorf("serialize query vector: %w", err)
202 }202 }
modified internal/cluster/jaccard.go +4 -4
@@ -10,9 +10,9 @@ import (
1010
1111 "github.com/hashicorp/golang-lru/v2/expirable"
1212
13- "pkg.rbrt.fr/glean/internal/ai"
1413 "pkg.rbrt.fr/glean/internal/db"
1514 "pkg.rbrt.fr/glean/internal/feedback"
15+ "pkg.rbrt.fr/glean/internal/ml"
1616 )
1717
1818 // Config controls weights used during similarity computation (feed similarity
@@ -46,8 +46,8 @@ type Engine struct {
4646 mu sync.Mutex
4747 config Config
4848
49- embedder ai.Embedder
50- llm ai.TextModel
49+ embedder ml.Embedder
50+ llm ml.TextModel
5151 feedback *feedback.Service
5252
5353 feedCache *expirable.LRU[string, []*FeedRecommendation]
@@ -60,7 +60,7 @@ type Engine struct {
6060 // recCacheSize is the maximum number of recommendations to cache per user.
6161 const recCacheSize = 512
6262
63-func NewEngine(sqlDB *sql.DB, articles *db.ArticleStore, embedder ai.Embedder, llm ai.TextModel, fb *feedback.Service, logger *slog.Logger, cacheTTL time.Duration, config Config) *Engine {
63+func NewEngine(sqlDB *sql.DB, articles *db.ArticleStore, embedder ml.Embedder, llm ml.TextModel, fb *feedback.Service, logger *slog.Logger, cacheTTL time.Duration, config Config) *Engine {
6464 return &Engine{
6565 db: sqlDB,
6666 articles: articles,
@@ -10,9 +10,9 @@ import (
10 10
11 "github.com/hashicorp/golang-lru/v2/expirable"11 "github.com/hashicorp/golang-lru/v2/expirable"
12 12
13- "pkg.rbrt.fr/glean/internal/ai"
14 "pkg.rbrt.fr/glean/internal/db"13 "pkg.rbrt.fr/glean/internal/db"
15 "pkg.rbrt.fr/glean/internal/feedback"14 "pkg.rbrt.fr/glean/internal/feedback"
15+ "pkg.rbrt.fr/glean/internal/ml"
16 )16 )
17 17
18 // Config controls weights used during similarity computation (feed similarity18 // Config controls weights used during similarity computation (feed similarity
@@ -46,8 +46,8 @@ type Engine struct {
46 mu sync.Mutex46 mu sync.Mutex
47 config Config47 config Config
48 48
49- embedder ai.Embedder49+ embedder ml.Embedder
50- llm ai.TextModel50+ llm ml.TextModel
51 feedback *feedback.Service51 feedback *feedback.Service
52 52
53 feedCache *expirable.LRU[string, []*FeedRecommendation]53 feedCache *expirable.LRU[string, []*FeedRecommendation]
@@ -60,7 +60,7 @@ type Engine struct {
60 // recCacheSize is the maximum number of recommendations to cache per user.60 // recCacheSize is the maximum number of recommendations to cache per user.
61 const recCacheSize = 51261 const recCacheSize = 512
62 62
63-func NewEngine(sqlDB *sql.DB, articles *db.ArticleStore, embedder ai.Embedder, llm ai.TextModel, fb *feedback.Service, logger *slog.Logger, cacheTTL time.Duration, config Config) *Engine {63+func NewEngine(sqlDB *sql.DB, articles *db.ArticleStore, embedder ml.Embedder, llm ml.TextModel, fb *feedback.Service, logger *slog.Logger, cacheTTL time.Duration, config Config) *Engine {
64 return &Engine{64 return &Engine{
65 db: sqlDB,65 db: sqlDB,
66 articles: articles,66 articles: articles,
modified internal/cluster/scoring.go +2 -2
@@ -7,8 +7,8 @@ import (
77 "strings"
88 "time"
99
10- "pkg.rbrt.fr/glean/internal/ai"
1110 "pkg.rbrt.fr/glean/internal/db"
11+ "pkg.rbrt.fr/glean/internal/ml"
1212 )
1313
1414 type FeedRecommendation struct {
@@ -391,7 +391,7 @@ func (e *Engine) coldStartFromEmbeddings(ctx context.Context, userDID string, li
391391 subSet[u] = true
392392 }
393393
394- queryBlob, err := ai.AvgEmbeddings(blobs, dim)
394+ queryBlob, err := ml.AvgEmbeddings(blobs, dim)
395395 if err != nil {
396396 return nil, fmt.Errorf("serialize query vector: %w", err)
397397 }
@@ -7,8 +7,8 @@ import (
7 "strings"7 "strings"
8 "time"8 "time"
9 9
10- "pkg.rbrt.fr/glean/internal/ai"
11 "pkg.rbrt.fr/glean/internal/db"10 "pkg.rbrt.fr/glean/internal/db"
11+ "pkg.rbrt.fr/glean/internal/ml"
12 )12 )
13 13
14 type FeedRecommendation struct {14 type FeedRecommendation struct {
@@ -391,7 +391,7 @@ func (e *Engine) coldStartFromEmbeddings(ctx context.Context, userDID string, li
391 subSet[u] = true391 subSet[u] = true
392 }392 }
393 393
394- queryBlob, err := ai.AvgEmbeddings(blobs, dim)394+ queryBlob, err := ml.AvgEmbeddings(blobs, dim)
395 if err != nil {395 if err != nil {
396 return nil, fmt.Errorf("serialize query vector: %w", err)396 return nil, fmt.Errorf("serialize query vector: %w", err)
397 }397 }
deleted internal/langdetect/langdetect_test.go +0 -29
deleted file mode 100644
@@ -1,29 +0,0 @@
1-package langdetect
2-
3-import (
4- "testing"
5-
6- "gotest.tools/v3/assert"
7-)
8-
9-func TestKnownLanguages(t *testing.T) {
10- langs := KnownLanguages()
11- assert.Assert(t, len(langs) > 0)
12- found := false
13- for _, l := range langs {
14- if l.Code == "en" {
15- found = true
16- assert.Equal(t, l.Name, "English")
17- }
18- }
19- assert.Assert(t, found)
20-}
21-
22-func TestIsKnown(t *testing.T) {
23- assert.Assert(t, IsKnown("en"))
24- assert.Assert(t, IsKnown("ja"))
25- assert.Assert(t, IsKnown("zh"))
26- assert.Assert(t, !IsKnown("xx"))
27- assert.Assert(t, !IsKnown("bamboo-based plastic"))
28- assert.Assert(t, !IsKnown(""))
29-}
deleted file mode 100644
@@ -1,29 +0,0 @@
1-package langdetect
2-
3-import (
4- "testing"
5-
6- "gotest.tools/v3/assert"
7-)
8-
9-func TestKnownLanguages(t *testing.T) {
10- langs := KnownLanguages()
11- assert.Assert(t, len(langs) > 0)
12- found := false
13- for _, l := range langs {
14- if l.Code == "en" {
15- found = true
16- assert.Equal(t, l.Name, "English")
17- }
18- }
19- assert.Assert(t, found)
20-}
21-
22-func TestIsKnown(t *testing.T) {
23- assert.Assert(t, IsKnown("en"))
24- assert.Assert(t, IsKnown("ja"))
25- assert.Assert(t, IsKnown("zh"))
26- assert.Assert(t, !IsKnown("xx"))
27- assert.Assert(t, !IsKnown("bamboo-based plastic"))
28- assert.Assert(t, !IsKnown(""))
29-}
renamed internal/ml/embed.go +1 -1
similarity index 99%
rename from internal/ai/embed.go
rename to internal/ml/embed.go
@@ -1,4 +1,4 @@
1-package ai
1+package ml
22
33 import (
44 "context"
similarity index 99%
rename from internal/ai/embed.go
rename to internal/ml/embed.go
@@ -1,4 +1,4 @@
1-package ai1+package ml
2 2
3 import (3 import (
4 "context"4 "context"
renamed internal/ml/langdetect.go +2 -2
similarity index 95%
rename from internal/langdetect/langdetect.go
rename to internal/ml/langdetect.go
@@ -1,4 +1,4 @@
1-package langdetect
1+package ml
22
33 type Language struct {
44 Code string
@@ -54,7 +54,7 @@ func init() {
5454 }
5555 }
5656
57-func IsKnown(code string) bool {
57+func IsKnownLanguage(code string) bool {
5858 return knownSet[code]
5959 }
6060
similarity index 95%
rename from internal/langdetect/langdetect.go
rename to internal/ml/langdetect.go
@@ -1,4 +1,4 @@
1-package langdetect1+package ml
2 2
3 type Language struct {3 type Language struct {
4 Code string4 Code string
@@ -54,7 +54,7 @@ func init() {
54 }54 }
55 }55 }
56 56
57-func IsKnown(code string) bool {57+func IsKnownLanguage(code string) bool {
58 return knownSet[code]58 return knownSet[code]
59 }59 }
60 60
added internal/ml/langdetect_test.go +29 -0
new file mode 100644
@@ -0,0 +1,29 @@
1+package ml
2+
3+import (
4+ "testing"
5+
6+ "gotest.tools/v3/assert"
7+)
8+
9+func TestKnownLanguages(t *testing.T) {
10+ langs := KnownLanguages()
11+ assert.Assert(t, len(langs) > 0)
12+ found := false
13+ for _, l := range langs {
14+ if l.Code == "en" {
15+ found = true
16+ assert.Equal(t, l.Name, "English")
17+ }
18+ }
19+ assert.Assert(t, found)
20+}
21+
22+func TestIsKnownLanguage(t *testing.T) {
23+ assert.Assert(t, IsKnownLanguage("en"))
24+ assert.Assert(t, IsKnownLanguage("ja"))
25+ assert.Assert(t, IsKnownLanguage("zh"))
26+ assert.Assert(t, !IsKnownLanguage("xx"))
27+ assert.Assert(t, !IsKnownLanguage("bamboo-based plastic"))
28+ assert.Assert(t, !IsKnownLanguage(""))
29+}
new file mode 100644
@@ -0,0 +1,29 @@
1+package ml
2+
3+import (
4+ "testing"
5+
6+ "gotest.tools/v3/assert"
7+)
8+
9+func TestKnownLanguages(t *testing.T) {
10+ langs := KnownLanguages()
11+ assert.Assert(t, len(langs) > 0)
12+ found := false
13+ for _, l := range langs {
14+ if l.Code == "en" {
15+ found = true
16+ assert.Equal(t, l.Name, "English")
17+ }
18+ }
19+ assert.Assert(t, found)
20+}
21+
22+func TestIsKnownLanguage(t *testing.T) {
23+ assert.Assert(t, IsKnownLanguage("en"))
24+ assert.Assert(t, IsKnownLanguage("ja"))
25+ assert.Assert(t, IsKnownLanguage("zh"))
26+ assert.Assert(t, !IsKnownLanguage("xx"))
27+ assert.Assert(t, !IsKnownLanguage("bamboo-based plastic"))
28+ assert.Assert(t, !IsKnownLanguage(""))
29+}
renamed internal/ml/llm.go +2 -4
similarity index 95%
rename from internal/ai/llm.go
rename to internal/ml/llm.go
@@ -1,4 +1,4 @@
1-package ai
1+package ml
22
33 import (
44 "context"
@@ -7,8 +7,6 @@ import (
77
88 "github.com/openai/openai-go"
99 "github.com/openai/openai-go/option"
10-
11- "pkg.rbrt.fr/glean/internal/langdetect"
1210 )
1311
1412 type TextModel interface {
@@ -72,7 +70,7 @@ func (c *llm) DetectLanguages(ctx context.Context, texts []string) ([]string, er
7270 code := strings.TrimSpace(line)
7371 code = strings.TrimPrefix(code, fmt.Sprintf("%d.", i+1))
7472 code = strings.TrimSpace(code)
75- if langdetect.IsKnown(code) {
73+ if IsKnownLanguage(code) {
7674 result[i] = code
7775 }
7876 }
similarity index 95%
rename from internal/ai/llm.go
rename to internal/ml/llm.go
@@ -1,4 +1,4 @@
1-package ai1+package ml
2 2
3 import (3 import (
4 "context"4 "context"
@@ -7,8 +7,6 @@ import (
7 7
8 "github.com/openai/openai-go"8 "github.com/openai/openai-go"
9 "github.com/openai/openai-go/option"9 "github.com/openai/openai-go/option"
10-
11- "pkg.rbrt.fr/glean/internal/langdetect"
12 )10 )
13 11
14 type TextModel interface {12 type TextModel interface {
@@ -72,7 +70,7 @@ func (c *llm) DetectLanguages(ctx context.Context, texts []string) ([]string, er
72 code := strings.TrimSpace(line)70 code := strings.TrimSpace(line)
73 code = strings.TrimPrefix(code, fmt.Sprintf("%d.", i+1))71 code = strings.TrimPrefix(code, fmt.Sprintf("%d.", i+1))
74 code = strings.TrimSpace(code)72 code = strings.TrimSpace(code)
75- if langdetect.IsKnown(code) {73+ if IsKnownLanguage(code) {
76 result[i] = code74 result[i] = code
77 }75 }
78 }76 }
modified internal/server/articles_handler.go +1 -2
@@ -14,7 +14,6 @@ import (
1414
1515 "pkg.rbrt.fr/glean/internal/atproto"
1616 "pkg.rbrt.fr/glean/internal/db"
17- "pkg.rbrt.fr/glean/internal/sanitize"
1817 )
1918
2019 func writeLikeButton(w http.ResponseWriter, articleID int64, liked bool, count int, bordered bool) {
@@ -468,7 +467,7 @@ func (s *Server) handleFetchContent(w http.ResponseWriter, r *http.Request) {
468467 return
469468 }
470469
471- cleaned := sanitize.HTML(content)
470+ cleaned := sanitizeHTML(content)
472471
473472 if err := s.dbs.Articles.UpdateArticleFullContent(ctx, id, cleaned); err != nil {
474473 s.logger.Error("failed to save full content", "error", err, "id", id)
@@ -14,7 +14,6 @@ import (
14 14
15 "pkg.rbrt.fr/glean/internal/atproto"15 "pkg.rbrt.fr/glean/internal/atproto"
16 "pkg.rbrt.fr/glean/internal/db"16 "pkg.rbrt.fr/glean/internal/db"
17- "pkg.rbrt.fr/glean/internal/sanitize"
18 )17 )
19 18
20 func writeLikeButton(w http.ResponseWriter, articleID int64, liked bool, count int, bordered bool) {19 func writeLikeButton(w http.ResponseWriter, articleID int64, liked bool, count int, bordered bool) {
@@ -468,7 +467,7 @@ func (s *Server) handleFetchContent(w http.ResponseWriter, r *http.Request) {
468 return467 return
469 }468 }
470 469
471- cleaned := sanitize.HTML(content)470+ cleaned := sanitizeHTML(content)
472 471
473 if err := s.dbs.Articles.UpdateArticleFullContent(ctx, id, cleaned); err != nil {472 if err := s.dbs.Articles.UpdateArticleFullContent(ctx, id, cleaned); err != nil {
474 s.logger.Error("failed to save full content", "error", err, "id", id)473 s.logger.Error("failed to save full content", "error", err, "id", id)
modified internal/server/profile_handler.go +2 -2
@@ -9,7 +9,7 @@ import (
99
1010 "pkg.rbrt.fr/glean/internal/atproto"
1111 "pkg.rbrt.fr/glean/internal/db"
12- "pkg.rbrt.fr/glean/internal/langdetect"
12+ "pkg.rbrt.fr/glean/internal/ml"
1313 )
1414
1515 func (s *Server) handleProfile(w http.ResponseWriter, r *http.Request) {
@@ -109,7 +109,7 @@ func (s *Server) handleProfile(w http.ResponseWriter, r *http.Request) {
109109 "SubscriptionCount": subCount,
110110 "AnnotationCount": len(annotations),
111111 "UserLanguages": userLangs,
112- "AvailableLanguages": langdetect.KnownLanguages(),
112+ "AvailableLanguages": ml.KnownLanguages(),
113113 "ExpandedView": userSettings.ExpandedView,
114114 })
115115 }
@@ -9,7 +9,7 @@ import (
9 9
10 "pkg.rbrt.fr/glean/internal/atproto"10 "pkg.rbrt.fr/glean/internal/atproto"
11 "pkg.rbrt.fr/glean/internal/db"11 "pkg.rbrt.fr/glean/internal/db"
12- "pkg.rbrt.fr/glean/internal/langdetect"12+ "pkg.rbrt.fr/glean/internal/ml"
13 )13 )
14 14
15 func (s *Server) handleProfile(w http.ResponseWriter, r *http.Request) {15 func (s *Server) handleProfile(w http.ResponseWriter, r *http.Request) {
@@ -109,7 +109,7 @@ func (s *Server) handleProfile(w http.ResponseWriter, r *http.Request) {
109 "SubscriptionCount": subCount,109 "SubscriptionCount": subCount,
110 "AnnotationCount": len(annotations),110 "AnnotationCount": len(annotations),
111 "UserLanguages": userLangs,111 "UserLanguages": userLangs,
112- "AvailableLanguages": langdetect.KnownLanguages(),112+ "AvailableLanguages": ml.KnownLanguages(),
113 "ExpandedView": userSettings.ExpandedView,113 "ExpandedView": userSettings.ExpandedView,
114 })114 })
115 }115 }
renamed internal/server/sanitize.go +3 -3
similarity index 97%
rename from internal/sanitize/sanitize.go
rename to internal/server/sanitize.go
@@ -1,4 +1,4 @@
1-package sanitize
1+package server
22
33 import (
44 "regexp"
@@ -52,7 +52,7 @@ func isAllowedIframe(tag string) bool {
5252 return false
5353 }
5454
55-func HTML(input string) string {
55+func sanitizeHTML(input string) string {
5656 s := input
5757 s = scriptRe.ReplaceAllString(s, "")
5858
@@ -97,7 +97,7 @@ var htmlEntities = map[string]string{
9797 "‘": "'", "”": "\"", "“": "\"",
9898 }
9999
100-func PlainText(input string) string {
100+func plainText(input string) string {
101101 s := tagRe.ReplaceAllString(input, " ")
102102 for entity, replacement := range htmlEntities {
103103 s = strings.ReplaceAll(s, entity, replacement)
similarity index 97%
rename from internal/sanitize/sanitize.go
rename to internal/server/sanitize.go
@@ -1,4 +1,4 @@
1-package sanitize1+package server
2 2
3 import (3 import (
4 "regexp"4 "regexp"
@@ -52,7 +52,7 @@ func isAllowedIframe(tag string) bool {
52 return false52 return false
53 }53 }
54 54
55-func HTML(input string) string {55+func sanitizeHTML(input string) string {
56 s := input56 s := input
57 s = scriptRe.ReplaceAllString(s, "")57 s = scriptRe.ReplaceAllString(s, "")
58 58
@@ -97,7 +97,7 @@ var htmlEntities = map[string]string{
97 "‘": "'", "”": "\"", "“": "\"",97 "‘": "'", "”": "\"", "“": "\"",
98 }98 }
99 99
100-func PlainText(input string) string {100+func plainText(input string) string {
101 s := tagRe.ReplaceAllString(input, " ")101 s := tagRe.ReplaceAllString(input, " ")
102 for entity, replacement := range htmlEntities {102 for entity, replacement := range htmlEntities {
103 s = strings.ReplaceAll(s, entity, replacement)103 s = strings.ReplaceAll(s, entity, replacement)
renamed internal/server/sanitize_test.go +47 -47
similarity index 69%
rename from internal/sanitize/sanitize_test.go
rename to internal/server/sanitize_test.go
@@ -1,13 +1,13 @@
1-package sanitize
1+package server
22
33 import (
44 "strings"
55 "testing"
66 )
77
8-func TestHTML_RemovesScriptTags(t *testing.T) {
8+func TestSanitizeHTML_RemovesScriptTags(t *testing.T) {
99 input := `<p>Hello</p><script>alert('xss')</script><p>World</p>`
10- got := HTML(input)
10+ got := sanitizeHTML(input)
1111 if strings.Contains(got, "<script") {
1212 t.Fatalf("script tag not removed: %s", got)
1313 }
@@ -16,17 +16,17 @@ func TestHTML_RemovesScriptTags(t *testing.T) {
1616 }
1717 }
1818
19-func TestHTML_RemovesEvilIframeTags(t *testing.T) {
19+func TestSanitizeHTML_RemovesEvilIframeTags(t *testing.T) {
2020 input := `<p>Hello</p><iframe src="evil.com"></iframe>`
21- got := HTML(input)
21+ got := sanitizeHTML(input)
2222 if strings.Contains(got, "<iframe") {
2323 t.Fatalf("iframe tag not removed: %s", got)
2424 }
2525 }
2626
27-func TestHTML_PreservesYouTubeIframe(t *testing.T) {
27+func TestSanitizeHTML_PreservesYouTubeIframe(t *testing.T) {
2828 input := `<iframe src="https://www.youtube.com/embed/dQw4w9WgXcQ" width="560" height="315"></iframe>`
29- got := HTML(input)
29+ got := sanitizeHTML(input)
3030 if !strings.Contains(got, "<iframe") {
3131 t.Fatalf("youtube iframe removed: %s", got)
3232 }
@@ -35,25 +35,25 @@ func TestHTML_PreservesYouTubeIframe(t *testing.T) {
3535 }
3636 }
3737
38-func TestHTML_PreservesVimeoIframe(t *testing.T) {
38+func TestSanitizeHTML_PreservesVimeoIframe(t *testing.T) {
3939 input := `<iframe src="https://player.vimeo.com/video/12345" width="640" height="360"></iframe>`
40- got := HTML(input)
40+ got := sanitizeHTML(input)
4141 if !strings.Contains(got, "<iframe") {
4242 t.Fatalf("vimeo iframe removed: %s", got)
4343 }
4444 }
4545
46-func TestHTML_PreservesSpotifyIframe(t *testing.T) {
46+func TestSanitizeHTML_PreservesSpotifyIframe(t *testing.T) {
4747 input := `<iframe src="https://open.spotify.com/embed/track/abc123" width="300" height="80"></iframe>`
48- got := HTML(input)
48+ got := sanitizeHTML(input)
4949 if !strings.Contains(got, "<iframe") {
5050 t.Fatalf("spotify iframe removed: %s", got)
5151 }
5252 }
5353
54-func TestHTML_RemovesEventHandler(t *testing.T) {
54+func TestSanitizeHTML_RemovesEventHandler(t *testing.T) {
5555 input := `<div onclick="alert('xss')">Hello</div>`
56- got := HTML(input)
56+ got := sanitizeHTML(input)
5757 if strings.Contains(got, "onclick") {
5858 t.Fatalf("onclick handler not removed: %s", got)
5959 }
@@ -62,111 +62,111 @@ func TestHTML_RemovesEventHandler(t *testing.T) {
6262 }
6363 }
6464
65-func TestHTML_RemovesJavascriptHref(t *testing.T) {
65+func TestSanitizeHTML_RemovesJavascriptHref(t *testing.T) {
6666 input := `<a href="javascript:alert('xss')">click</a>`
67- got := HTML(input)
67+ got := sanitizeHTML(input)
6868 if strings.Contains(got, "javascript:") {
6969 t.Fatalf("javascript: href not removed: %s", got)
7070 }
7171 }
7272
73-func TestHTML_RemovesObjectTags(t *testing.T) {
73+func TestSanitizeHTML_RemovesObjectTags(t *testing.T) {
7474 input := `<object data="evil.swf"></object>`
75- got := HTML(input)
75+ got := sanitizeHTML(input)
7676 if strings.Contains(got, "<object") {
7777 t.Fatalf("object tag not removed: %s", got)
7878 }
7979 }
8080
81-func TestHTML_RemovesFormTags(t *testing.T) {
81+func TestSanitizeHTML_RemovesFormTags(t *testing.T) {
8282 input := `<form action="evil.com"><input type="submit"></form>`
83- got := HTML(input)
83+ got := sanitizeHTML(input)
8484 if strings.Contains(got, "<form") {
8585 t.Fatalf("form tag not removed: %s", got)
8686 }
8787 }
8888
89-func TestHTML_RemovesMetaTags(t *testing.T) {
89+func TestSanitizeHTML_RemovesMetaTags(t *testing.T) {
9090 input := `<meta http-equiv="refresh" content="0;url=evil.com">`
91- got := HTML(input)
91+ got := sanitizeHTML(input)
9292 if strings.Contains(got, "<meta") {
9393 t.Fatalf("meta tag not removed: %s", got)
9494 }
9595 }
9696
97-func TestHTML_RemovesBaseTags(t *testing.T) {
97+func TestSanitizeHTML_RemovesBaseTags(t *testing.T) {
9898 input := `<base href="evil.com">`
99- got := HTML(input)
99+ got := sanitizeHTML(input)
100100 if strings.Contains(got, "<base") {
101101 t.Fatalf("base tag not removed: %s", got)
102102 }
103103 }
104104
105-func TestHTML_RemovesStyleExpression(t *testing.T) {
105+func TestSanitizeHTML_RemovesStyleExpression(t *testing.T) {
106106 input := `<div style="background: expression(alert('xss'))">Hello</div>`
107- got := HTML(input)
107+ got := sanitizeHTML(input)
108108 if strings.Contains(got, "expression") {
109109 t.Fatalf("expression not removed: %s", got)
110110 }
111111 }
112112
113-func TestHTML_PreservesSafeContent(t *testing.T) {
113+func TestSanitizeHTML_PreservesSafeContent(t *testing.T) {
114114 input := `<h1>Title</h1><p>Paragraph with <strong>bold</strong> and <em>italic</em>.</p><ul><li>item</li></ul>`
115- got := HTML(input)
115+ got := sanitizeHTML(input)
116116 if got != input {
117117 t.Fatalf("safe content modified:\ngot: %s\nwant: %s", got, input)
118118 }
119119 }
120120
121-func TestHTML_PreservesImages(t *testing.T) {
121+func TestSanitizeHTML_PreservesImages(t *testing.T) {
122122 input := `<img src="photo.jpg" alt="photo">`
123- got := HTML(input)
123+ got := sanitizeHTML(input)
124124 if got != input {
125125 t.Fatalf("img tag modified: %s", got)
126126 }
127127 }
128128
129-func TestHTML_PreservesLinks(t *testing.T) {
129+func TestSanitizeHTML_PreservesLinks(t *testing.T) {
130130 input := `<a href="https://example.com">link</a>`
131- got := HTML(input)
131+ got := sanitizeHTML(input)
132132 if got != input {
133133 t.Fatalf("link modified: %s", got)
134134 }
135135 }
136136
137-func TestHTML_HandlesCaseInsensitiveScript(t *testing.T) {
137+func TestSanitizeHTML_HandlesCaseInsensitiveScript(t *testing.T) {
138138 input := `<SCRIPT>alert('xss')</SCRIPT>`
139- got := HTML(input)
139+ got := sanitizeHTML(input)
140140 if strings.Contains(got, "<SCRIPT") {
141141 t.Fatalf("case-insensitive script not removed: %s", got)
142142 }
143143 }
144144
145-func TestHTML_HandlesMultilineScript(t *testing.T) {
145+func TestSanitizeHTML_HandlesMultilineScript(t *testing.T) {
146146 input := "<script>\nalert('xss');\n</script>"
147- got := HTML(input)
147+ got := sanitizeHTML(input)
148148 if strings.Contains(got, "<script") {
149149 t.Fatalf("multiline script not removed: %s", got)
150150 }
151151 }
152152
153-func TestHTML_RemovesOnEventHandlers(t *testing.T) {
153+func TestSanitizeHTML_RemovesOnEventHandlers(t *testing.T) {
154154 cases := []string{
155155 `<div onmouseover="alert(1)">`,
156156 `<img onerror="alert(1)" src="x">`,
157157 `<body onload="alert(1)">`,
158158 }
159159 for _, input := range cases {
160- got := HTML(input)
160+ got := sanitizeHTML(input)
161161 if strings.Contains(got, " on") {
162162 t.Fatalf("event handler not removed from %q: %s", input, got)
163163 }
164164 }
165165 }
166166
167-func TestHTML_ConvertsYouTubeLink(t *testing.T) {
167+func TestSanitizeHTML_ConvertsYouTubeLink(t *testing.T) {
168168 input := `<p>Check this out:</p><a href="https://www.youtube.com/watch?v=dQw4w9WgXcQ">Watch on YouTube</a>`
169- got := HTML(input)
169+ got := sanitizeHTML(input)
170170 if !strings.Contains(got, `<iframe src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ"`) {
171171 t.Fatalf("youtube link not converted to iframe: %s", got)
172172 }
@@ -175,33 +175,33 @@ func TestHTML_ConvertsYouTubeLink(t *testing.T) {
175175 }
176176 }
177177
178-func TestHTML_ConvertsYoutuBeLink(t *testing.T) {
178+func TestSanitizeHTML_ConvertsYoutuBeLink(t *testing.T) {
179179 input := `<a href="https://youtu.be/dQw4w9WgXcQ">Watch</a>`
180- got := HTML(input)
180+ got := sanitizeHTML(input)
181181 if !strings.Contains(got, `<iframe src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ"`) {
182182 t.Fatalf("youtu.be link not converted to iframe: %s", got)
183183 }
184184 }
185185
186-func TestHTML_ConvertsYouTubeShortsLink(t *testing.T) {
186+func TestSanitizeHTML_ConvertsYouTubeShortsLink(t *testing.T) {
187187 input := `<a href="https://www.youtube.com/shorts/abc12345678">Short</a>`
188- got := HTML(input)
188+ got := sanitizeHTML(input)
189189 if !strings.Contains(got, `<iframe src="https://www.youtube-nocookie.com/embed/abc12345678"`) {
190190 t.Fatalf("youtube shorts link not converted to iframe: %s", got)
191191 }
192192 }
193193
194-func TestHTML_ConvertsVimeoLink(t *testing.T) {
194+func TestSanitizeHTML_ConvertsVimeoLink(t *testing.T) {
195195 input := `<a href="https://vimeo.com/123456789">Watch on Vimeo</a>`
196- got := HTML(input)
196+ got := sanitizeHTML(input)
197197 if !strings.Contains(got, `<iframe src="https://player.vimeo.com/video/123456789"`) {
198198 t.Fatalf("vimeo link not converted to iframe: %s", got)
199199 }
200200 }
201201
202-func TestHTML_PreservesNonMediaLinks(t *testing.T) {
202+func TestSanitizeHTML_PreservesNonMediaLinks(t *testing.T) {
203203 input := `<a href="https://example.com/article">Read more</a>`
204- got := HTML(input)
204+ got := sanitizeHTML(input)
205205 if !strings.Contains(got, `<a href="https://example.com/article">Read more</a>`) {
206206 t.Fatalf("non-media link was modified: %s", got)
207207 }
similarity index 69%
rename from internal/sanitize/sanitize_test.go
rename to internal/server/sanitize_test.go
@@ -1,13 +1,13 @@
1-package sanitize1+package server
2 2
3 import (3 import (
4 "strings"4 "strings"
5 "testing"5 "testing"
6 )6 )
7 7
8-func TestHTML_RemovesScriptTags(t *testing.T) {8+func TestSanitizeHTML_RemovesScriptTags(t *testing.T) {
9 input := `<p>Hello</p><script>alert('xss')</script><p>World</p>`9 input := `<p>Hello</p><script>alert('xss')</script><p>World</p>`
10- got := HTML(input)10+ got := sanitizeHTML(input)
11 if strings.Contains(got, "<script") {11 if strings.Contains(got, "<script") {
12 t.Fatalf("script tag not removed: %s", got)12 t.Fatalf("script tag not removed: %s", got)
13 }13 }
@@ -16,17 +16,17 @@ func TestHTML_RemovesScriptTags(t *testing.T) {
16 }16 }
17 }17 }
18 18
19-func TestHTML_RemovesEvilIframeTags(t *testing.T) {19+func TestSanitizeHTML_RemovesEvilIframeTags(t *testing.T) {
20 input := `<p>Hello</p><iframe src="evil.com"></iframe>`20 input := `<p>Hello</p><iframe src="evil.com"></iframe>`
21- got := HTML(input)21+ got := sanitizeHTML(input)
22 if strings.Contains(got, "<iframe") {22 if strings.Contains(got, "<iframe") {
23 t.Fatalf("iframe tag not removed: %s", got)23 t.Fatalf("iframe tag not removed: %s", got)
24 }24 }
25 }25 }
26 26
27-func TestHTML_PreservesYouTubeIframe(t *testing.T) {27+func TestSanitizeHTML_PreservesYouTubeIframe(t *testing.T) {
28 input := `<iframe src="https://www.youtube.com/embed/dQw4w9WgXcQ" width="560" height="315"></iframe>`28 input := `<iframe src="https://www.youtube.com/embed/dQw4w9WgXcQ" width="560" height="315"></iframe>`
29- got := HTML(input)29+ got := sanitizeHTML(input)
30 if !strings.Contains(got, "<iframe") {30 if !strings.Contains(got, "<iframe") {
31 t.Fatalf("youtube iframe removed: %s", got)31 t.Fatalf("youtube iframe removed: %s", got)
32 }32 }
@@ -35,25 +35,25 @@ func TestHTML_PreservesYouTubeIframe(t *testing.T) {
35 }35 }
36 }36 }
37 37
38-func TestHTML_PreservesVimeoIframe(t *testing.T) {38+func TestSanitizeHTML_PreservesVimeoIframe(t *testing.T) {
39 input := `<iframe src="https://player.vimeo.com/video/12345" width="640" height="360"></iframe>`39 input := `<iframe src="https://player.vimeo.com/video/12345" width="640" height="360"></iframe>`
40- got := HTML(input)40+ got := sanitizeHTML(input)
41 if !strings.Contains(got, "<iframe") {41 if !strings.Contains(got, "<iframe") {
42 t.Fatalf("vimeo iframe removed: %s", got)42 t.Fatalf("vimeo iframe removed: %s", got)
43 }43 }
44 }44 }
45 45
46-func TestHTML_PreservesSpotifyIframe(t *testing.T) {46+func TestSanitizeHTML_PreservesSpotifyIframe(t *testing.T) {
47 input := `<iframe src="https://open.spotify.com/embed/track/abc123" width="300" height="80"></iframe>`47 input := `<iframe src="https://open.spotify.com/embed/track/abc123" width="300" height="80"></iframe>`
48- got := HTML(input)48+ got := sanitizeHTML(input)
49 if !strings.Contains(got, "<iframe") {49 if !strings.Contains(got, "<iframe") {
50 t.Fatalf("spotify iframe removed: %s", got)50 t.Fatalf("spotify iframe removed: %s", got)
51 }51 }
52 }52 }
53 53
54-func TestHTML_RemovesEventHandler(t *testing.T) {54+func TestSanitizeHTML_RemovesEventHandler(t *testing.T) {
55 input := `<div onclick="alert('xss')">Hello</div>`55 input := `<div onclick="alert('xss')">Hello</div>`
56- got := HTML(input)56+ got := sanitizeHTML(input)
57 if strings.Contains(got, "onclick") {57 if strings.Contains(got, "onclick") {
58 t.Fatalf("onclick handler not removed: %s", got)58 t.Fatalf("onclick handler not removed: %s", got)
59 }59 }
@@ -62,111 +62,111 @@ func TestHTML_RemovesEventHandler(t *testing.T) {
62 }62 }
63 }63 }
64 64
65-func TestHTML_RemovesJavascriptHref(t *testing.T) {65+func TestSanitizeHTML_RemovesJavascriptHref(t *testing.T) {
66 input := `<a href="javascript:alert('xss')">click</a>`66 input := `<a href="javascript:alert('xss')">click</a>`
67- got := HTML(input)67+ got := sanitizeHTML(input)
68 if strings.Contains(got, "javascript:") {68 if strings.Contains(got, "javascript:") {
69 t.Fatalf("javascript: href not removed: %s", got)69 t.Fatalf("javascript: href not removed: %s", got)
70 }70 }
71 }71 }
72 72
73-func TestHTML_RemovesObjectTags(t *testing.T) {73+func TestSanitizeHTML_RemovesObjectTags(t *testing.T) {
74 input := `<object data="evil.swf"></object>`74 input := `<object data="evil.swf"></object>`
75- got := HTML(input)75+ got := sanitizeHTML(input)
76 if strings.Contains(got, "<object") {76 if strings.Contains(got, "<object") {
77 t.Fatalf("object tag not removed: %s", got)77 t.Fatalf("object tag not removed: %s", got)
78 }78 }
79 }79 }
80 80
81-func TestHTML_RemovesFormTags(t *testing.T) {81+func TestSanitizeHTML_RemovesFormTags(t *testing.T) {
82 input := `<form action="evil.com"><input type="submit"></form>`82 input := `<form action="evil.com"><input type="submit"></form>`
83- got := HTML(input)83+ got := sanitizeHTML(input)
84 if strings.Contains(got, "<form") {84 if strings.Contains(got, "<form") {
85 t.Fatalf("form tag not removed: %s", got)85 t.Fatalf("form tag not removed: %s", got)
86 }86 }
87 }87 }
88 88
89-func TestHTML_RemovesMetaTags(t *testing.T) {89+func TestSanitizeHTML_RemovesMetaTags(t *testing.T) {
90 input := `<meta http-equiv="refresh" content="0;url=evil.com">`90 input := `<meta http-equiv="refresh" content="0;url=evil.com">`
91- got := HTML(input)91+ got := sanitizeHTML(input)
92 if strings.Contains(got, "<meta") {92 if strings.Contains(got, "<meta") {
93 t.Fatalf("meta tag not removed: %s", got)93 t.Fatalf("meta tag not removed: %s", got)
94 }94 }
95 }95 }
96 96
97-func TestHTML_RemovesBaseTags(t *testing.T) {97+func TestSanitizeHTML_RemovesBaseTags(t *testing.T) {
98 input := `<base href="evil.com">`98 input := `<base href="evil.com">`
99- got := HTML(input)99+ got := sanitizeHTML(input)
100 if strings.Contains(got, "<base") {100 if strings.Contains(got, "<base") {
101 t.Fatalf("base tag not removed: %s", got)101 t.Fatalf("base tag not removed: %s", got)
102 }102 }
103 }103 }
104 104
105-func TestHTML_RemovesStyleExpression(t *testing.T) {105+func TestSanitizeHTML_RemovesStyleExpression(t *testing.T) {
106 input := `<div style="background: expression(alert('xss'))">Hello</div>`106 input := `<div style="background: expression(alert('xss'))">Hello</div>`
107- got := HTML(input)107+ got := sanitizeHTML(input)
108 if strings.Contains(got, "expression") {108 if strings.Contains(got, "expression") {
109 t.Fatalf("expression not removed: %s", got)109 t.Fatalf("expression not removed: %s", got)
110 }110 }
111 }111 }
112 112
113-func TestHTML_PreservesSafeContent(t *testing.T) {113+func TestSanitizeHTML_PreservesSafeContent(t *testing.T) {
114 input := `<h1>Title</h1><p>Paragraph with <strong>bold</strong> and <em>italic</em>.</p><ul><li>item</li></ul>`114 input := `<h1>Title</h1><p>Paragraph with <strong>bold</strong> and <em>italic</em>.</p><ul><li>item</li></ul>`
115- got := HTML(input)115+ got := sanitizeHTML(input)
116 if got != input {116 if got != input {
117 t.Fatalf("safe content modified:\ngot: %s\nwant: %s", got, input)117 t.Fatalf("safe content modified:\ngot: %s\nwant: %s", got, input)
118 }118 }
119 }119 }
120 120
121-func TestHTML_PreservesImages(t *testing.T) {121+func TestSanitizeHTML_PreservesImages(t *testing.T) {
122 input := `<img src="photo.jpg" alt="photo">`122 input := `<img src="photo.jpg" alt="photo">`
123- got := HTML(input)123+ got := sanitizeHTML(input)
124 if got != input {124 if got != input {
125 t.Fatalf("img tag modified: %s", got)125 t.Fatalf("img tag modified: %s", got)
126 }126 }
127 }127 }
128 128
129-func TestHTML_PreservesLinks(t *testing.T) {129+func TestSanitizeHTML_PreservesLinks(t *testing.T) {
130 input := `<a href="https://example.com">link</a>`130 input := `<a href="https://example.com">link</a>`
131- got := HTML(input)131+ got := sanitizeHTML(input)
132 if got != input {132 if got != input {
133 t.Fatalf("link modified: %s", got)133 t.Fatalf("link modified: %s", got)
134 }134 }
135 }135 }
136 136
137-func TestHTML_HandlesCaseInsensitiveScript(t *testing.T) {137+func TestSanitizeHTML_HandlesCaseInsensitiveScript(t *testing.T) {
138 input := `<SCRIPT>alert('xss')</SCRIPT>`138 input := `<SCRIPT>alert('xss')</SCRIPT>`
139- got := HTML(input)139+ got := sanitizeHTML(input)
140 if strings.Contains(got, "<SCRIPT") {140 if strings.Contains(got, "<SCRIPT") {
141 t.Fatalf("case-insensitive script not removed: %s", got)141 t.Fatalf("case-insensitive script not removed: %s", got)
142 }142 }
143 }143 }
144 144
145-func TestHTML_HandlesMultilineScript(t *testing.T) {145+func TestSanitizeHTML_HandlesMultilineScript(t *testing.T) {
146 input := "<script>\nalert('xss');\n</script>"146 input := "<script>\nalert('xss');\n</script>"
147- got := HTML(input)147+ got := sanitizeHTML(input)
148 if strings.Contains(got, "<script") {148 if strings.Contains(got, "<script") {
149 t.Fatalf("multiline script not removed: %s", got)149 t.Fatalf("multiline script not removed: %s", got)
150 }150 }
151 }151 }
152 152
153-func TestHTML_RemovesOnEventHandlers(t *testing.T) {153+func TestSanitizeHTML_RemovesOnEventHandlers(t *testing.T) {
154 cases := []string{154 cases := []string{
155 `<div onmouseover="alert(1)">`,155 `<div onmouseover="alert(1)">`,
156 `<img onerror="alert(1)" src="x">`,156 `<img onerror="alert(1)" src="x">`,
157 `<body onload="alert(1)">`,157 `<body onload="alert(1)">`,
158 }158 }
159 for _, input := range cases {159 for _, input := range cases {
160- got := HTML(input)160+ got := sanitizeHTML(input)
161 if strings.Contains(got, " on") {161 if strings.Contains(got, " on") {
162 t.Fatalf("event handler not removed from %q: %s", input, got)162 t.Fatalf("event handler not removed from %q: %s", input, got)
163 }163 }
164 }164 }
165 }165 }
166 166
167-func TestHTML_ConvertsYouTubeLink(t *testing.T) {167+func TestSanitizeHTML_ConvertsYouTubeLink(t *testing.T) {
168 input := `<p>Check this out:</p><a href="https://www.youtube.com/watch?v=dQw4w9WgXcQ">Watch on YouTube</a>`168 input := `<p>Check this out:</p><a href="https://www.youtube.com/watch?v=dQw4w9WgXcQ">Watch on YouTube</a>`
169- got := HTML(input)169+ got := sanitizeHTML(input)
170 if !strings.Contains(got, `<iframe src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ"`) {170 if !strings.Contains(got, `<iframe src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ"`) {
171 t.Fatalf("youtube link not converted to iframe: %s", got)171 t.Fatalf("youtube link not converted to iframe: %s", got)
172 }172 }
@@ -175,33 +175,33 @@ func TestHTML_ConvertsYouTubeLink(t *testing.T) {
175 }175 }
176 }176 }
177 177
178-func TestHTML_ConvertsYoutuBeLink(t *testing.T) {178+func TestSanitizeHTML_ConvertsYoutuBeLink(t *testing.T) {
179 input := `<a href="https://youtu.be/dQw4w9WgXcQ">Watch</a>`179 input := `<a href="https://youtu.be/dQw4w9WgXcQ">Watch</a>`
180- got := HTML(input)180+ got := sanitizeHTML(input)
181 if !strings.Contains(got, `<iframe src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ"`) {181 if !strings.Contains(got, `<iframe src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ"`) {
182 t.Fatalf("youtu.be link not converted to iframe: %s", got)182 t.Fatalf("youtu.be link not converted to iframe: %s", got)
183 }183 }
184 }184 }
185 185
186-func TestHTML_ConvertsYouTubeShortsLink(t *testing.T) {186+func TestSanitizeHTML_ConvertsYouTubeShortsLink(t *testing.T) {
187 input := `<a href="https://www.youtube.com/shorts/abc12345678">Short</a>`187 input := `<a href="https://www.youtube.com/shorts/abc12345678">Short</a>`
188- got := HTML(input)188+ got := sanitizeHTML(input)
189 if !strings.Contains(got, `<iframe src="https://www.youtube-nocookie.com/embed/abc12345678"`) {189 if !strings.Contains(got, `<iframe src="https://www.youtube-nocookie.com/embed/abc12345678"`) {
190 t.Fatalf("youtube shorts link not converted to iframe: %s", got)190 t.Fatalf("youtube shorts link not converted to iframe: %s", got)
191 }191 }
192 }192 }
193 193
194-func TestHTML_ConvertsVimeoLink(t *testing.T) {194+func TestSanitizeHTML_ConvertsVimeoLink(t *testing.T) {
195 input := `<a href="https://vimeo.com/123456789">Watch on Vimeo</a>`195 input := `<a href="https://vimeo.com/123456789">Watch on Vimeo</a>`
196- got := HTML(input)196+ got := sanitizeHTML(input)
197 if !strings.Contains(got, `<iframe src="https://player.vimeo.com/video/123456789"`) {197 if !strings.Contains(got, `<iframe src="https://player.vimeo.com/video/123456789"`) {
198 t.Fatalf("vimeo link not converted to iframe: %s", got)198 t.Fatalf("vimeo link not converted to iframe: %s", got)
199 }199 }
200 }200 }
201 201
202-func TestHTML_PreservesNonMediaLinks(t *testing.T) {202+func TestSanitizeHTML_PreservesNonMediaLinks(t *testing.T) {
203 input := `<a href="https://example.com/article">Read more</a>`203 input := `<a href="https://example.com/article">Read more</a>`
204- got := HTML(input)204+ got := sanitizeHTML(input)
205 if !strings.Contains(got, `<a href="https://example.com/article">Read more</a>`) {205 if !strings.Contains(got, `<a href="https://example.com/article">Read more</a>`) {
206 t.Fatalf("non-media link was modified: %s", got)206 t.Fatalf("non-media link was modified: %s", got)
207 }207 }
modified internal/server/server.go +2 -3
@@ -28,7 +28,6 @@ import (
2828 "pkg.rbrt.fr/glean/internal/feed"
2929 "pkg.rbrt.fr/glean/internal/feedback"
3030 "pkg.rbrt.fr/glean/internal/metrics"
31- "pkg.rbrt.fr/glean/internal/sanitize"
3231 "pkg.rbrt.fr/glean/internal/scraper"
3332 "pkg.rbrt.fr/glean/internal/tmpl"
3433 "pkg.rbrt.fr/glean/static"
@@ -334,9 +333,9 @@ func (s *Server) loadTemplates() {
334333 }, host)
335334 },
336335 "sanitizeHTML": func(input string) template.HTML {
337- return template.HTML(sanitize.HTML(input))
336+ return template.HTML(sanitizeHTML(input))
338337 },
339- "plainText": sanitize.PlainText,
338+ "plainText": plainText,
340339 "now": time.Now,
341340 "activeClass": func(activePath, linkPath string) string {
342341 if activePath == linkPath || (len(activePath) > len(linkPath) && activePath[:len(linkPath)+1] == linkPath+"/") {
@@ -28,7 +28,6 @@ import (
28 "pkg.rbrt.fr/glean/internal/feed"28 "pkg.rbrt.fr/glean/internal/feed"
29 "pkg.rbrt.fr/glean/internal/feedback"29 "pkg.rbrt.fr/glean/internal/feedback"
30 "pkg.rbrt.fr/glean/internal/metrics"30 "pkg.rbrt.fr/glean/internal/metrics"
31- "pkg.rbrt.fr/glean/internal/sanitize"
32 "pkg.rbrt.fr/glean/internal/scraper"31 "pkg.rbrt.fr/glean/internal/scraper"
33 "pkg.rbrt.fr/glean/internal/tmpl"32 "pkg.rbrt.fr/glean/internal/tmpl"
34 "pkg.rbrt.fr/glean/static"33 "pkg.rbrt.fr/glean/static"
@@ -334,9 +333,9 @@ func (s *Server) loadTemplates() {
334 }, host)333 }, host)
335 },334 },
336 "sanitizeHTML": func(input string) template.HTML {335 "sanitizeHTML": func(input string) template.HTML {
337- return template.HTML(sanitize.HTML(input))336+ return template.HTML(sanitizeHTML(input))
338 },337 },
339- "plainText": sanitize.PlainText,338+ "plainText": plainText,
340 "now": time.Now,339 "now": time.Now,
341 "activeClass": func(activePath, linkPath string) string {340 "activeClass": func(activePath, linkPath string) string {
342 if activePath == linkPath || (len(activePath) > len(linkPath) && activePath[:len(linkPath)+1] == linkPath+"/") {341 if activePath == linkPath || (len(activePath) > len(linkPath) && activePath[:len(linkPath)+1] == linkPath+"/") {
modified internal/server/settings_handler.go +2 -2
@@ -4,7 +4,7 @@ import (
44 "net/http"
55 "strings"
66
7- "pkg.rbrt.fr/glean/internal/langdetect"
7+ "pkg.rbrt.fr/glean/internal/ml"
88 )
99
1010 func (s *Server) handleUpdateLanguages(w http.ResponseWriter, r *http.Request) {
@@ -16,7 +16,7 @@ func (s *Server) handleUpdateLanguages(w http.ResponseWriter, r *http.Request) {
1616 }
1717
1818 valid := make(map[string]bool)
19- for _, known := range langdetect.KnownLanguages() {
19+ for _, known := range ml.KnownLanguages() {
2020 valid[known.Code] = true
2121 }
2222
@@ -4,7 +4,7 @@ import (
4 "net/http"4 "net/http"
5 "strings"5 "strings"
6 6
7- "pkg.rbrt.fr/glean/internal/langdetect"7+ "pkg.rbrt.fr/glean/internal/ml"
8 )8 )
9 9
10 func (s *Server) handleUpdateLanguages(w http.ResponseWriter, r *http.Request) {10 func (s *Server) handleUpdateLanguages(w http.ResponseWriter, r *http.Request) {
@@ -16,7 +16,7 @@ func (s *Server) handleUpdateLanguages(w http.ResponseWriter, r *http.Request) {
16 }16 }
17 17
18 valid := make(map[string]bool)18 valid := make(map[string]bool)
19- for _, known := range langdetect.KnownLanguages() {19+ for _, known := range ml.KnownLanguages() {
20 valid[known.Code] = true20 valid[known.Code] = true
21 }21 }
22 22
modified main.go +5 -5
@@ -13,12 +13,12 @@ import (
1313 "syscall"
1414 "time"
1515
16- "pkg.rbrt.fr/glean/internal/ai"
1716 "pkg.rbrt.fr/glean/internal/atproto"
1817 "pkg.rbrt.fr/glean/internal/cluster"
1918 "pkg.rbrt.fr/glean/internal/db"
2019 "pkg.rbrt.fr/glean/internal/feed"
2120 "pkg.rbrt.fr/glean/internal/feedback"
21+ "pkg.rbrt.fr/glean/internal/ml"
2222 "pkg.rbrt.fr/glean/internal/server"
2323
2424 vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
@@ -75,9 +75,9 @@ func main() {
7575 siteFetcher := atproto.NewStandardSiteFetcher(logger)
7676 scheduler := feed.NewScheduler(storeAdapter, siteFetcher, logger, *fetchInterval, 30*time.Minute)
7777
78- var embedder ai.Embedder
78+ var embedder ml.Embedder
7979 if embedURL := envOr("GLEAN_EMBED_BASE_URL", ""); embedURL != "" {
80- embedder = ai.NewEmbedder(ai.EmbedConfig{
80+ embedder = ml.NewEmbedder(ml.EmbedConfig{
8181 BaseURL: embedURL,
8282 APIKey: envOr("GLEAN_EMBED_API_KEY", ""),
8383 Model: envOr("GLEAN_EMBED_MODEL", "text-embedding-3-small"),
@@ -92,9 +92,9 @@ func main() {
9292 }
9393 }
9494
95- var llm ai.TextModel
95+ var llm ml.TextModel
9696 if llmURL := envOr("GLEAN_LLM_BASE_URL", ""); llmURL != "" {
97- llm = ai.NewLLM(ai.LLMConfig{
97+ llm = ml.NewLLM(ml.LLMConfig{
9898 BaseURL: llmURL,
9999 APIKey: envOr("GLEAN_LLM_API_KEY", ""),
100100 Model: envOr("GLEAN_LLM_MODEL", "gpt-4o-mini"),
@@ -13,12 +13,12 @@ import (
13 "syscall"13 "syscall"
14 "time"14 "time"
15 15
16- "pkg.rbrt.fr/glean/internal/ai"
17 "pkg.rbrt.fr/glean/internal/atproto"16 "pkg.rbrt.fr/glean/internal/atproto"
18 "pkg.rbrt.fr/glean/internal/cluster"17 "pkg.rbrt.fr/glean/internal/cluster"
19 "pkg.rbrt.fr/glean/internal/db"18 "pkg.rbrt.fr/glean/internal/db"
20 "pkg.rbrt.fr/glean/internal/feed"19 "pkg.rbrt.fr/glean/internal/feed"
21 "pkg.rbrt.fr/glean/internal/feedback"20 "pkg.rbrt.fr/glean/internal/feedback"
21+ "pkg.rbrt.fr/glean/internal/ml"
22 "pkg.rbrt.fr/glean/internal/server"22 "pkg.rbrt.fr/glean/internal/server"
23 23
24 vec "github.com/asg017/sqlite-vec-go-bindings/cgo"24 vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
@@ -75,9 +75,9 @@ func main() {
75 siteFetcher := atproto.NewStandardSiteFetcher(logger)75 siteFetcher := atproto.NewStandardSiteFetcher(logger)
76 scheduler := feed.NewScheduler(storeAdapter, siteFetcher, logger, *fetchInterval, 30*time.Minute)76 scheduler := feed.NewScheduler(storeAdapter, siteFetcher, logger, *fetchInterval, 30*time.Minute)
77 77
78- var embedder ai.Embedder78+ var embedder ml.Embedder
79 if embedURL := envOr("GLEAN_EMBED_BASE_URL", ""); embedURL != "" {79 if embedURL := envOr("GLEAN_EMBED_BASE_URL", ""); embedURL != "" {
80- embedder = ai.NewEmbedder(ai.EmbedConfig{80+ embedder = ml.NewEmbedder(ml.EmbedConfig{
81 BaseURL: embedURL,81 BaseURL: embedURL,
82 APIKey: envOr("GLEAN_EMBED_API_KEY", ""),82 APIKey: envOr("GLEAN_EMBED_API_KEY", ""),
83 Model: envOr("GLEAN_EMBED_MODEL", "text-embedding-3-small"),83 Model: envOr("GLEAN_EMBED_MODEL", "text-embedding-3-small"),
@@ -92,9 +92,9 @@ func main() {
92 }92 }
93 }93 }
94 94
95- var llm ai.TextModel95+ var llm ml.TextModel
96 if llmURL := envOr("GLEAN_LLM_BASE_URL", ""); llmURL != "" {96 if llmURL := envOr("GLEAN_LLM_BASE_URL", ""); llmURL != "" {
97- llm = ai.NewLLM(ai.LLMConfig{97+ llm = ml.NewLLM(ml.LLMConfig{
98 BaseURL: llmURL,98 BaseURL: llmURL,
99 APIKey: envOr("GLEAN_LLM_API_KEY", ""),99 APIKey: envOr("GLEAN_LLM_API_KEY", ""),
100 Model: envOr("GLEAN_LLM_MODEL", "gpt-4o-mini"),100 Model: envOr("GLEAN_LLM_MODEL", "gpt-4o-mini"),