rickub
nandi/gleanpublic⑂ Fork 0
⑂ dd385c8
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Implement dead link stripping and custom 404 pageUnverified

Julien Robert committed 2026-04-21T12:37:05+02:00 Browse files
dd385c8 parent: 6691ef3
modified .gitignore +1 -1
@@ -24,7 +24,7 @@ go.work.sum
2424
2525 # database
2626 *.db
27-*.db-journal
27+*.db-*
2828
2929 # tailwind
3030 static/output.css
@@ -24,7 +24,7 @@ go.work.sum
24 24
25 # database25 # database
26 *.db26 *.db
27-*.db-journal27+*.db-*
28 28
29 # tailwind29 # tailwind
30 static/output.css30 static/output.css
modified internal/db/social.go +1 -1
@@ -33,7 +33,7 @@ type Like struct {
3333
3434 func (db *DB) CreateAnnotation(ctx context.Context, a *Annotation) error {
3535 _, err := db.ExecContext(ctx, `
36- INSERT INTO annotations (uri, author_did, feed_url, article_url, quote, note, tags, rating, created_at, cid)
36+ INSERT OR IGNORE INTO annotations (uri, author_did, feed_url, article_url, quote, note, tags, rating, created_at, cid)
3737 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
3838 `, a.URI, a.AuthorDID, a.FeedURL, a.ArticleURL, a.Quote, a.Note, a.Tags, a.Rating, a.CreatedAt, a.CID)
3939 return err
@@ -33,7 +33,7 @@ type Like struct {
33 33
34 func (db *DB) CreateAnnotation(ctx context.Context, a *Annotation) error {34 func (db *DB) CreateAnnotation(ctx context.Context, a *Annotation) error {
35 _, err := db.ExecContext(ctx, `35 _, err := db.ExecContext(ctx, `
36- INSERT INTO annotations (uri, author_did, feed_url, article_url, quote, note, tags, rating, created_at, cid)36+ INSERT OR IGNORE INTO annotations (uri, author_did, feed_url, article_url, quote, note, tags, rating, created_at, cid)
37 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)37 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
38 `, a.URI, a.AuthorDID, a.FeedURL, a.ArticleURL, a.Quote, a.Note, a.Tags, a.Rating, a.CreatedAt, a.CID)38 `, a.URI, a.AuthorDID, a.FeedURL, a.ArticleURL, a.Quote, a.Note, a.Tags, a.Rating, a.CreatedAt, a.CID)
39 return err39 return err
modified internal/scraper/scraper.go +37 -0
@@ -245,6 +245,15 @@ func renderNode(n *html.Node) string {
245245 return
246246 }
247247 if node.Type == html.ElementNode {
248+ if node.Data == "a" && isDeadLink(node) {
249+ for c := node.FirstChild; c != nil; c = c.NextSibling {
250+ write(c)
251+ }
252+ return
253+ }
254+ if node.Data == "img" && isDeadImage(node) {
255+ return
256+ }
248257 buf.WriteString("<")
249258 buf.WriteString(node.Data)
250259 for _, attr := range node.Attr {
@@ -279,3 +288,31 @@ func isVoidElement(tag string) bool {
279288 }
280289 return false
281290 }
291+
292+func isDeadLink(n *html.Node) bool {
293+ for _, attr := range n.Attr {
294+ if attr.Key != "href" {
295+ continue
296+ }
297+ href := strings.TrimSpace(attr.Val)
298+ if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
299+ return false
300+ }
301+ return true
302+ }
303+ return true
304+}
305+
306+func isDeadImage(n *html.Node) bool {
307+ for _, attr := range n.Attr {
308+ if attr.Key != "src" {
309+ continue
310+ }
311+ src := strings.TrimSpace(attr.Val)
312+ if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
313+ return false
314+ }
315+ return true
316+ }
317+ return true
318+}
@@ -245,6 +245,15 @@ func renderNode(n *html.Node) string {
245 return245 return
246 }246 }
247 if node.Type == html.ElementNode {247 if node.Type == html.ElementNode {
248+ if node.Data == "a" && isDeadLink(node) {
249+ for c := node.FirstChild; c != nil; c = c.NextSibling {
250+ write(c)
251+ }
252+ return
253+ }
254+ if node.Data == "img" && isDeadImage(node) {
255+ return
256+ }
248 buf.WriteString("<")257 buf.WriteString("<")
249 buf.WriteString(node.Data)258 buf.WriteString(node.Data)
250 for _, attr := range node.Attr {259 for _, attr := range node.Attr {
@@ -279,3 +288,31 @@ func isVoidElement(tag string) bool {
279 }288 }
280 return false289 return false
281 }290 }
291+
292+func isDeadLink(n *html.Node) bool {
293+ for _, attr := range n.Attr {
294+ if attr.Key != "href" {
295+ continue
296+ }
297+ href := strings.TrimSpace(attr.Val)
298+ if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
299+ return false
300+ }
301+ return true
302+ }
303+ return true
304+}
305+
306+func isDeadImage(n *html.Node) bool {
307+ for _, attr := range n.Attr {
308+ if attr.Key != "src" {
309+ continue
310+ }
311+ src := strings.TrimSpace(attr.Val)
312+ if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
313+ return false
314+ }
315+ return true
316+ }
317+ return true
318+}
modified internal/scraper/scraper_test.go +41 -1
@@ -132,7 +132,7 @@ func TestScrape_FallsBackToArchive(t *testing.T) {
132132 func TestRenderNode_VoidElements(t *testing.T) {
133133 html := `<!DOCTYPE html><html><body>
134134 <article>
135- <p>Text with <br>break and <img src="test.jpg"> image</p>
135+ <p>Text with <br>break and <img src="https://example.com/test.jpg"> image</p>
136136 </article>
137137 </body></html>`
138138
@@ -142,3 +142,43 @@ func TestRenderNode_VoidElements(t *testing.T) {
142142 assert.Assert(t, strings.Contains(content, "<img"))
143143 assert.Assert(t, !strings.Contains(content, "</br>"))
144144 }
145+
146+func TestRenderNode_StripsDeadLinks(t *testing.T) {
147+ html := `<!DOCTYPE html><html><body>
148+ <article>
149+ <p>Read <a href="/">home</a> and <a href="/about">about</a> and <a>no href</a> and <a href="#section">jump</a> and <a href="javascript:void(0)">click</a> and <a href="mailto:test@example.com">email</a> and <a href="https://example.com/article">real link</a> and <a href="http://example.com/page">http link</a>.</p>
150+ </article>
151+ </body></html>`
152+
153+ content, err := extractContent(strings.NewReader(html))
154+ assert.NilError(t, err)
155+ assert.Assert(t, strings.Contains(content, "home"))
156+ assert.Assert(t, strings.Contains(content, "about"))
157+ assert.Assert(t, strings.Contains(content, "no href"))
158+ assert.Assert(t, strings.Contains(content, "jump"))
159+ assert.Assert(t, strings.Contains(content, "click"))
160+ assert.Assert(t, strings.Contains(content, "email"))
161+ assert.Assert(t, strings.Contains(content, "real link"))
162+ assert.Assert(t, strings.Contains(content, "http link"))
163+ assert.Assert(t, !strings.Contains(content, `href="/"`))
164+ assert.Assert(t, !strings.Contains(content, `href="/about"`))
165+ assert.Assert(t, !strings.Contains(content, `href="#section"`))
166+ assert.Assert(t, !strings.Contains(content, `href="javascript:`))
167+ assert.Assert(t, !strings.Contains(content, `href="mailto:`))
168+ assert.Assert(t, strings.Contains(content, `<a href="https://example.com/article">real link</a>`))
169+ assert.Assert(t, strings.Contains(content, `<a href="http://example.com/page">http link</a>`))
170+}
171+
172+func TestRenderNode_StripsDeadImages(t *testing.T) {
173+ html := `<!DOCTYPE html><html><body>
174+ <article>
175+ <p>Text <img src="/images/photo.jpg"> more text <img src="https://example.com/img.png"> end <img src="data:image/png;base64,abc"> <img> </p>
176+ </article>
177+ </body></html>`
178+
179+ content, err := extractContent(strings.NewReader(html))
180+ assert.NilError(t, err)
181+ assert.Assert(t, !strings.Contains(content, `/images/photo.jpg`))
182+ assert.Assert(t, !strings.Contains(content, `data:image`))
183+ assert.Assert(t, strings.Contains(content, `<img src="https://example.com/img.png"`))
184+}
@@ -132,7 +132,7 @@ func TestScrape_FallsBackToArchive(t *testing.T) {
132 func TestRenderNode_VoidElements(t *testing.T) {132 func TestRenderNode_VoidElements(t *testing.T) {
133 html := `<!DOCTYPE html><html><body>133 html := `<!DOCTYPE html><html><body>
134 <article>134 <article>
135- <p>Text with <br>break and <img src="test.jpg"> image</p>135+ <p>Text with <br>break and <img src="https://example.com/test.jpg"> image</p>
136 </article>136 </article>
137 </body></html>`137 </body></html>`
138 138
@@ -142,3 +142,43 @@ func TestRenderNode_VoidElements(t *testing.T) {
142 assert.Assert(t, strings.Contains(content, "<img"))142 assert.Assert(t, strings.Contains(content, "<img"))
143 assert.Assert(t, !strings.Contains(content, "</br>"))143 assert.Assert(t, !strings.Contains(content, "</br>"))
144 }144 }
145+
146+func TestRenderNode_StripsDeadLinks(t *testing.T) {
147+ html := `<!DOCTYPE html><html><body>
148+ <article>
149+ <p>Read <a href="/">home</a> and <a href="/about">about</a> and <a>no href</a> and <a href="#section">jump</a> and <a href="javascript:void(0)">click</a> and <a href="mailto:test@example.com">email</a> and <a href="https://example.com/article">real link</a> and <a href="http://example.com/page">http link</a>.</p>
150+ </article>
151+ </body></html>`
152+
153+ content, err := extractContent(strings.NewReader(html))
154+ assert.NilError(t, err)
155+ assert.Assert(t, strings.Contains(content, "home"))
156+ assert.Assert(t, strings.Contains(content, "about"))
157+ assert.Assert(t, strings.Contains(content, "no href"))
158+ assert.Assert(t, strings.Contains(content, "jump"))
159+ assert.Assert(t, strings.Contains(content, "click"))
160+ assert.Assert(t, strings.Contains(content, "email"))
161+ assert.Assert(t, strings.Contains(content, "real link"))
162+ assert.Assert(t, strings.Contains(content, "http link"))
163+ assert.Assert(t, !strings.Contains(content, `href="/"`))
164+ assert.Assert(t, !strings.Contains(content, `href="/about"`))
165+ assert.Assert(t, !strings.Contains(content, `href="#section"`))
166+ assert.Assert(t, !strings.Contains(content, `href="javascript:`))
167+ assert.Assert(t, !strings.Contains(content, `href="mailto:`))
168+ assert.Assert(t, strings.Contains(content, `<a href="https://example.com/article">real link</a>`))
169+ assert.Assert(t, strings.Contains(content, `<a href="http://example.com/page">http link</a>`))
170+}
171+
172+func TestRenderNode_StripsDeadImages(t *testing.T) {
173+ html := `<!DOCTYPE html><html><body>
174+ <article>
175+ <p>Text <img src="/images/photo.jpg"> more text <img src="https://example.com/img.png"> end <img src="data:image/png;base64,abc"> <img> </p>
176+ </article>
177+ </body></html>`
178+
179+ content, err := extractContent(strings.NewReader(html))
180+ assert.NilError(t, err)
181+ assert.Assert(t, !strings.Contains(content, `/images/photo.jpg`))
182+ assert.Assert(t, !strings.Contains(content, `data:image`))
183+ assert.Assert(t, strings.Contains(content, `<img src="https://example.com/img.png"`))
184+}
modified internal/server/server.go +6 -0
@@ -204,6 +204,7 @@ func (s *Server) setupRoutes() {
204204
205205 s.router.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.FS(static.Files))))
206206 s.router.Handle("/metrics", promhttp.Handler())
207+ s.router.NotFound(s.handleNotFound)
207208 }
208209
209210 func (s *Server) loadTemplates() {
@@ -432,6 +433,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
432433 s.router.ServeHTTP(w, r)
433434 }
434435
436+func (s *Server) handleNotFound(w http.ResponseWriter, r *http.Request) {
437+ w.WriteHeader(http.StatusNotFound)
438+ s.render(w, r, "404.html", nil)
439+}
440+
435441 func (s *Server) render(w http.ResponseWriter, r *http.Request, name string, data map[string]any) {
436442 if data == nil {
437443 data = map[string]any{}
@@ -204,6 +204,7 @@ func (s *Server) setupRoutes() {
204 204
205 s.router.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.FS(static.Files))))205 s.router.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.FS(static.Files))))
206 s.router.Handle("/metrics", promhttp.Handler())206 s.router.Handle("/metrics", promhttp.Handler())
207+ s.router.NotFound(s.handleNotFound)
207 }208 }
208 209
209 func (s *Server) loadTemplates() {210 func (s *Server) loadTemplates() {
@@ -432,6 +433,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
432 s.router.ServeHTTP(w, r)433 s.router.ServeHTTP(w, r)
433 }434 }
434 435
436+func (s *Server) handleNotFound(w http.ResponseWriter, r *http.Request) {
437+ w.WriteHeader(http.StatusNotFound)
438+ s.render(w, r, "404.html", nil)
439+}
440+
435 func (s *Server) render(w http.ResponseWriter, r *http.Request, name string, data map[string]any) {441 func (s *Server) render(w http.ResponseWriter, r *http.Request, name string, data map[string]any) {
436 if data == nil {442 if data == nil {
437 data = map[string]any{}443 data = map[string]any{}
added internal/tmpl/404.html +14 -0
new file mode 100644
@@ -0,0 +1,14 @@
1+{{define "404.html"}}
2+<div class="max-w-md mx-auto mt-20">
3+ <div class="bg-spot-surface rounded-xl shadow-spot-heavy p-8">
4+ <div class="flex justify-center mb-6">
5+ <span class="w-14 h-14">{{template "logo-icon"}}</span>
6+ </div>
7+ <h1 class="text-2xl font-bold text-spot-text mb-2 text-center">Page not found</h1>
8+ <p class="text-spot-secondary text-sm mb-6 text-center">This page doesn't exist, but your feeds do.</p>
9+ <a href="/dashboard" class="w-full flex items-center justify-center gap-2 bg-spot-green text-white rounded-pill px-4 py-3 text-sm font-bold uppercase tracking-button hover:brightness-110 transition">
10+ Back to Dashboard
11+ </a>
12+ </div>
13+</div>
14+{{end}}
new file mode 100644
@@ -0,0 +1,14 @@
1+{{define "404.html"}}
2+<div class="max-w-md mx-auto mt-20">
3+ <div class="bg-spot-surface rounded-xl shadow-spot-heavy p-8">
4+ <div class="flex justify-center mb-6">
5+ <span class="w-14 h-14">{{template "logo-icon"}}</span>
6+ </div>
7+ <h1 class="text-2xl font-bold text-spot-text mb-2 text-center">Page not found</h1>
8+ <p class="text-spot-secondary text-sm mb-6 text-center">This page doesn't exist, but your feeds do.</p>
9+ <a href="/dashboard" class="w-full flex items-center justify-center gap-2 bg-spot-green text-white rounded-pill px-4 py-3 text-sm font-bold uppercase tracking-button hover:brightness-110 transition">
10+ Back to Dashboard
11+ </a>
12+ </div>
13+</div>
14+{{end}}
modified internal/tmpl/base.html +1 -0
@@ -314,6 +314,7 @@
314314
315315 document.addEventListener('keydown', function(e) {
316316 if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return;
317+ if (e.ctrlKey || e.metaKey || e.altKey) return;
317318 if (e.key === 'g') window.location.href = '/dashboard';
318319 if (e.key === 'a') window.location.href = '/articles';
319320 if (e.key === 'f') window.location.href = '/feeds';
@@ -314,6 +314,7 @@
314 314
315 document.addEventListener('keydown', function(e) {315 document.addEventListener('keydown', function(e) {
316 if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return;316 if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return;
317+ if (e.ctrlKey || e.metaKey || e.altKey) return;
317 if (e.key === 'g') window.location.href = '/dashboard';318 if (e.key === 'g') window.location.href = '/dashboard';
318 if (e.key === 'a') window.location.href = '/articles';319 if (e.key === 'a') window.location.href = '/articles';
319 if (e.key === 'f') window.location.href = '/feeds';320 if (e.key === 'f') window.location.href = '/feeds';