nandi/glean · 9a6ff22

modified docs/specs.md +0 -1 Viewed

@@ -844,7 +844,6 @@ The server renders HTML fragments that htmx swaps into the page. No JSON API nee
844	844	\| `/dashboard` \| GET \| Main dashboard: unread articles, recommendations sidebar \|
845	845	\| `/feeds` \| GET \| Manage RSS subscriptions (OPML import for onboarding) \|
846	846	\| `/feeds/list` \| GET \| Feed list fragment (htmx partial) \|
847		-\| `/feeds/discover-url` \| GET \| Discover feed URL from a website \|
848	847	\| `/feeds/opml/upload` \| POST \| Upload OPML file to bulk-import subscriptions \|
849	848	\| `/feeds/opml/download` \| GET \| Export subscriptions as OPML (offboarding) \|
850	849	\| `/feeds/add` \| POST \| Add a single feed URL \|

  | `/dashboard`                   | GET    | Main dashboard: unread articles, recommendations sidebar |
  | `/feeds`                       | GET    | Manage RSS subscriptions (OPML import for onboarding)    |
  | `/feeds/list`                  | GET    | Feed list fragment (htmx partial)                        |
--| `/feeds/discover-url`          | GET    | Discover feed URL from a website                         |
  | `/feeds/opml/upload`           | POST   | Upload OPML file to bulk-import subscriptions            |
  | `/feeds/opml/download`         | GET    | Export subscriptions as OPML (offboarding)               |
  | `/feeds/add`                   | POST   | Add a single feed URL                                    |

modified internal/cluster/scoring.go +1 -1 Viewed

@@ -224,7 +224,7 @@ func (e *Engine) ComputeArticleRecommendationsOnDemand(ctx context.Context, user
224	224	JOIN articles a ON a.feed_url = la.feed_url AND a.url = la.article_url
225	225	LEFT JOIN feeds f ON f.feed_url = la.feed_url
226	226	LEFT JOIN social_likes sl ON sl.feed_url = la.feed_url AND sl.article_url = la.article_url
227		- ORDER BY score DESC
	227	+ ORDER BY score DESC, a.published DESC
228	228	LIMIT ?
229	229	`, userDID, userDID, userDID, userDID, userDID, userDID, w.WLike, w.WSocial, limit)
230	230	if err != nil {

  		JOIN articles a ON a.feed_url = la.feed_url AND a.url = la.article_url
  		LEFT JOIN feeds f ON f.feed_url = la.feed_url
  		LEFT JOIN social_likes sl ON sl.feed_url = la.feed_url AND sl.article_url = la.article_url
--		ORDER BY score DESC
++		ORDER BY score DESC, a.published DESC
  		LIMIT ?
  	`, userDID, userDID, userDID, userDID, userDID, userDID, w.WLike, w.WSocial, limit)
  	if err != nil {

modified internal/db/article.go +52 -0 Viewed

@@ -6,6 +6,8 @@ import (
6	6	"strings"
7	7	"time"
8	8	"unicode"
	9	+
	10	+ "pkg.rbrt.fr/glean/internal/feed"
9	11	)
10	12
11	13	type Article struct {
@@ -52,6 +54,56 @@ func (db DB) UpsertArticle(ctx context.Context, article Article) (int64, error
52	54	return id, err
53	55	}
54	56
	57	+func (db *DB) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
	58	+ if len(articles) == 0 {
	59	+ return nil
	60	+ }
	61	+
	62	+ err := upsertArticlesBatch(ctx, db, articles)
	63	+ if err != nil {
	64	+ err = upsertArticlesBatch(ctx, db, articles)
	65	+ }
	66	+ return err
	67	+}
	68	+
	69	+func upsertArticlesBatch(ctx context.Context, db *DB, articles []feed.Article) error {
	70	+ tx, err := db.BeginTx(ctx, nil)
	71	+ if err != nil {
	72	+ return err
	73	+ }
	74	+ defer tx.Rollback()
	75	+
	76	+ stmt, err := tx.PrepareContext(ctx, `
	77	+ INSERT INTO articles (feed_url, guid, title, url, author, summary, content, published, updated)
	78	+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
	79	+ ON CONFLICT(feed_url, guid) DO NOTHING
	80	+ `)
	81	+ if err != nil {
	82	+ return err
	83	+ }
	84	+ defer stmt.Close()
	85	+
	86	+ for _, a := range articles {
	87	+ url := sql.NullString{String: a.URL, Valid: a.URL != ""}
	88	+ author := sql.NullString{String: a.Author, Valid: a.Author != ""}
	89	+ summary := sql.NullString{String: a.Summary, Valid: a.Summary != ""}
	90	+ content := sql.NullString{String: a.Content, Valid: a.Content != ""}
	91	+ var published, updated sql.NullTime
	92	+ if !a.Published.IsZero() {
	93	+ published = sql.NullTime{Time: a.Published, Valid: true}
	94	+ }
	95	+ if !a.Updated.IsZero() {
	96	+ updated = sql.NullTime{Time: a.Updated, Valid: true}
	97	+ }
	98	+
	99	+ if _, err := stmt.ExecContext(ctx, a.FeedURL, a.GUID, a.Title, url, author, summary, content, published, updated); err != nil {
	100	+ return err
	101	+ }
	102	+ }
	103	+
	104	+ return tx.Commit()
	105	+}
	106	+
55	107	func (db DB) GetArticle(ctx context.Context, id int64) (Article, error) {
56	108	a := &Article{}
57	109	err := db.QueryRowContext(ctx, `

  	"strings"
  	"time"
  	"unicode"
++
++	"pkg.rbrt.fr/glean/internal/feed"
  )
  type Article struct {
  	return id, err
  }
++func (db *DB) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
++	if len(articles) == 0 {
++		return nil
++	}
++
++	err := upsertArticlesBatch(ctx, db, articles)
++	if err != nil {
++		err = upsertArticlesBatch(ctx, db, articles)
++	}
++	return err
++}
++
++func upsertArticlesBatch(ctx context.Context, db *DB, articles []feed.Article) error {
++	tx, err := db.BeginTx(ctx, nil)
++	if err != nil {
++		return err
++	}
++	defer tx.Rollback()
++
++	stmt, err := tx.PrepareContext(ctx, `
++		INSERT INTO articles (feed_url, guid, title, url, author, summary, content, published, updated)
++		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
++		ON CONFLICT(feed_url, guid) DO NOTHING
++	`)
++	if err != nil {
++		return err
++	}
++	defer stmt.Close()
++
++	for _, a := range articles {
++		url := sql.NullString{String: a.URL, Valid: a.URL != ""}
++		author := sql.NullString{String: a.Author, Valid: a.Author != ""}
++		summary := sql.NullString{String: a.Summary, Valid: a.Summary != ""}
++		content := sql.NullString{String: a.Content, Valid: a.Content != ""}
++		var published, updated sql.NullTime
++		if !a.Published.IsZero() {
++			published = sql.NullTime{Time: a.Published, Valid: true}
++		}
++		if !a.Updated.IsZero() {
++			updated = sql.NullTime{Time: a.Updated, Valid: true}
++		}
++
++		if _, err := stmt.ExecContext(ctx, a.FeedURL, a.GUID, a.Title, url, author, summary, content, published, updated); err != nil {
++			return err
++		}
++	}
++
++	return tx.Commit()
++}
++
  func (db *DB) GetArticle(ctx context.Context, id int64) (*Article, error) {
  	a := &Article{}
  	err := db.QueryRowContext(ctx, `

modified internal/db/social.go +2 -2 Viewed

@@ -255,7 +255,7 @@ func (db *DB) ListTrendingArticlesForUser(ctx context.Context, userDID, since st
255	255	UNION SELECT f.target_did FROM follows f WHERE f.user_did = ?
256	256	)
257	257	GROUP BY ar.id
258		- ORDER BY like_count DESC, annotation_count DESC
	258	+ ORDER BY like_count DESC, annotation_count DESC, ar.published DESC
259	259	LIMIT ? OFFSET ?
260	260	`, since, userDID, since, userDID, userDID, userDID, userDID, userDID, limit, offset)
261	261	if err != nil {
@@ -291,7 +291,7 @@ func (db *DB) ListTrendingArticles(ctx context.Context, userDID, since string, l
291	291	LEFT JOIN likes ul ON ul.feed_url = l.feed_url AND ul.article_url = l.article_url AND ul.author_did = ?
292	292	WHERE l.created_at >= ?
293	293	GROUP BY ar.id
294		- ORDER BY like_count DESC, annotation_count DESC
	294	+ ORDER BY like_count DESC, annotation_count DESC, ar.published DESC
295	295	LIMIT ? OFFSET ?
296	296	`, since, userDID, since, limit, offset)
297	297	if err != nil {

  		    UNION SELECT f.target_did FROM follows f WHERE f.user_did = ?
  		  )
  		GROUP BY ar.id
--		ORDER BY like_count DESC, annotation_count DESC
++		ORDER BY like_count DESC, annotation_count DESC, ar.published DESC
  		LIMIT ? OFFSET ?
  	`, since, userDID, since, userDID, userDID, userDID, userDID, userDID, limit, offset)
  	if err != nil {
  		LEFT JOIN likes ul ON ul.feed_url = l.feed_url AND ul.article_url = l.article_url AND ul.author_did = ?
  		WHERE l.created_at >= ?
  		GROUP BY ar.id
--		ORDER BY like_count DESC, annotation_count DESC
++		ORDER BY like_count DESC, annotation_count DESC, ar.published DESC
  		LIMIT ? OFFSET ?
  	`, since, userDID, since, limit, offset)
  	if err != nil {

modified internal/db/store.go +4 -0 Viewed

@@ -56,6 +56,10 @@ func (a FeedStoreAdapter) UpsertArticle(ctx context.Context, article feed.Arti
56	56	return a.db.UpsertArticle(ctx, dbArticle)
57	57	}
58	58
	59	+func (a *FeedStoreAdapter) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
	60	+ return a.db.UpsertArticlesBatch(ctx, articles)
	61	+}
	62	+
59	63	func (a *FeedStoreAdapter) MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error {
60	64	return a.db.MarkFeedFetched(ctx, feedURL, etag, lastModified)
61	65	}

  	return a.db.UpsertArticle(ctx, dbArticle)
  }
++func (a *FeedStoreAdapter) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
++	return a.db.UpsertArticlesBatch(ctx, articles)
++}
++
  func (a *FeedStoreAdapter) MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error {
  	return a.db.MarkFeedFetched(ctx, feedURL, etag, lastModified)
  }

modified internal/feed/discover.go +34 -13 Viewed

@@ -116,21 +116,42 @@ func findFavicon(ctx context.Context, base *url.URL, links []string) string {
116	116	origin.RawQuery = ""
117	117	origin.Fragment = ""
118	118
	119	+ type result struct {
	120	+ url string
	121	+ found bool
	122	+ }
	123	+ found := make(chan result, 1)
	124	+ ctx, cancel := context.WithCancel(ctx)
	125	+ defer cancel()
	126	+
119	127	for _, path := range faviconPaths {
120		- u, _ := url.Parse(path)
121		- resolved := origin.ResolveReference(u)
122		- req, err := http.NewRequestWithContext(ctx, http.MethodGet, resolved.String(), nil)
123		- if err != nil {
124		- continue
125		- }
126		- resp, err := discoverClient.Do(req)
127		- if err != nil {
128		- continue
129		- }
130		- resp.Body.Close()
131		- if resp.StatusCode == http.StatusOK && imageContentTypes.matches(resp.Header.Get("Content-Type")) {
132		- return cleanFavicon(resolved.String())
	128	+ go func(path string) {
	129	+ u, _ := url.Parse(path)
	130	+ resolved := origin.ResolveReference(u)
	131	+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, resolved.String(), nil)
	132	+ if err != nil {
	133	+ return
	134	+ }
	135	+ resp, err := discoverClient.Do(req)
	136	+ if err != nil {
	137	+ return
	138	+ }
	139	+ resp.Body.Close()
	140	+ if resp.StatusCode == http.StatusOK && imageContentTypes.matches(resp.Header.Get("Content-Type")) {
	141	+ select {
	142	+ case found <- result{url: cleanFavicon(resolved.String()), found: true}:
	143	+ default:
	144	+ }
	145	+ }
	146	+ }(path)
	147	+ }
	148	+
	149	+ select {
	150	+ case r := <-found:
	151	+ if r.found {
	152	+ return r.url
133	153	}
	154	+ case <-ctx.Done():
134	155	}
135	156	return ""
136	157	}

  	origin.RawQuery = ""
  	origin.Fragment = ""
++	type result struct {
++		url  string
++		found bool
++	}
++	found := make(chan result, 1)
++	ctx, cancel := context.WithCancel(ctx)
++	defer cancel()
++
  	for _, path := range faviconPaths {
--		u, _ := url.Parse(path)
++		go func(path string) {
--		resolved := origin.ResolveReference(u)
++			u, _ := url.Parse(path)
--		req, err := http.NewRequestWithContext(ctx, http.MethodGet, resolved.String(), nil)
++			resolved := origin.ResolveReference(u)
--		if err != nil {
++			req, err := http.NewRequestWithContext(ctx, http.MethodGet, resolved.String(), nil)
--			continue
++			if err != nil {
--		}
++				return
--		resp, err := discoverClient.Do(req)
++			}
--		if err != nil {
++			resp, err := discoverClient.Do(req)
--			continue
++			if err != nil {
--		}
++				return
--		resp.Body.Close()
++			}
--		if resp.StatusCode == http.StatusOK && imageContentTypes.matches(resp.Header.Get("Content-Type")) {
++			resp.Body.Close()
--			return cleanFavicon(resolved.String())
++			if resp.StatusCode == http.StatusOK && imageContentTypes.matches(resp.Header.Get("Content-Type")) {
++				select {
++				case found <- result{url: cleanFavicon(resolved.String()), found: true}:
++				default:
++				}
++			}
++		}(path)
++	}
++
++	select {
++	case r := <-found:
++		if r.found {
++			return r.url
  		}
++	case <-ctx.Done():
  	}
  	return ""
  }

modified internal/feed/fetcher.go +9 -8 Viewed

@@ -70,6 +70,7 @@ func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string)
70	70	type FeedStore interface {
71	71	GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error)
72	72	UpsertArticle(ctx context.Context, article *Article) (int64, error)
	73	+ UpsertArticlesBatch(ctx context.Context, articles []Article) error
73	74	MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error
74	75	MarkFeedFetchError(ctx context.Context, feedURL, lastError string) error
75	76	UpdateFeedFavicon(ctx context.Context, feedURL, faviconURL string) error
@@ -120,7 +121,7 @@ func (s *Scheduler) fetchAll(ctx context.Context) {
120	121	return
121	122	}
122	123
123		- sem := make(chan struct{}, 3)
	124	+ sem := make(chan struct{}, 10)
124	125	var wg sync.WaitGroup
125	126	for _, f := range feeds {
126	127	wg.Add(1)
@@ -169,13 +170,13 @@ func (s Scheduler) FetchFeed(ctx context.Context, feed Feed) {
169	170
170	171	metrics.FeedsFetched.WithLabelValues("success").Inc()
171	172
172		- for i := range result.Articles {
173		- result.Articles[i].FeedURL = feed.URL
174		- if _, upsertErr := s.store.UpsertArticle(ctx, &result.Articles[i]); upsertErr != nil {
175		- s.logger.Error("failed to upsert article", "error", upsertErr, "url", result.Articles[i].URL)
176		- } else {
177		- metrics.ArticlesUpserted.Inc()
178		- }
	173	+ for _, article := range result.Articles {
	174	+ article.FeedURL = feed.URL
	175	+ }
	176	+ if err := s.store.UpsertArticlesBatch(ctx, result.Articles); err != nil {
	177	+ s.logger.Error("failed to upsert articles", "error", err, "feed", feed.URL)
	178	+ } else {
	179	+ metrics.ArticlesUpserted.Add(float64(len(result.Articles)))
179	180	}
180	181
181	182	if err := s.store.MarkFeedFetched(ctx, feed.URL, newEtag, newLastModified); err != nil {

  type FeedStore interface {
  	GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error)
  	UpsertArticle(ctx context.Context, article *Article) (int64, error)
++	UpsertArticlesBatch(ctx context.Context, articles []Article) error
  	MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error
  	MarkFeedFetchError(ctx context.Context, feedURL, lastError string) error
  	UpdateFeedFavicon(ctx context.Context, feedURL, faviconURL string) error
  		return
  	}
--	sem := make(chan struct{}, 3)
++	sem := make(chan struct{}, 10)
  	var wg sync.WaitGroup
  	for _, f := range feeds {
  		wg.Add(1)
  	metrics.FeedsFetched.WithLabelValues("success").Inc()
--	for i := range result.Articles {
++	for _, article := range result.Articles {
--		result.Articles[i].FeedURL = feed.URL
++		article.FeedURL = feed.URL
--		if _, upsertErr := s.store.UpsertArticle(ctx, &result.Articles[i]); upsertErr != nil {
++	}
--			s.logger.Error("failed to upsert article", "error", upsertErr, "url", result.Articles[i].URL)
++	if err := s.store.UpsertArticlesBatch(ctx, result.Articles); err != nil {
--		} else {
++		s.logger.Error("failed to upsert articles", "error", err, "feed", feed.URL)
--			metrics.ArticlesUpserted.Inc()
++	} else {
--		}
++		metrics.ArticlesUpserted.Add(float64(len(result.Articles)))
  	}
  	if err := s.store.MarkFeedFetched(ctx, feed.URL, newEtag, newLastModified); err != nil {

modified internal/server/feeds_handler.go +0 -26 Viewed

@@ -3,7 +3,6 @@ package server
3	3	import (
4	4	"context"
5	5	"database/sql"
6		- "encoding/json"
7	6	"errors"
8	7	"net/http"
9	8	"time"
@@ -374,31 +373,6 @@ func (s Server) handleRetryFeed(w http.ResponseWriter, r http.Request) {
374	373	})
375	374	}
376	375
377		-func (s Server) handleDiscoverFeedURL(w http.ResponseWriter, r http.Request) {
378		- siteURL := r.URL.Query().Get("url")
379		- if siteURL == "" {
380		- http.Error(w, "url required", http.StatusBadRequest)
381		- return
382		- }
383		-
384		- result, err := feed.Discover(r.Context(), siteURL)
385		- if err != nil {
386		- s.logger.Error("feed discovery failed", "error", err, "url", siteURL)
387		- http.Error(w, err.Error(), http.StatusInternalServerError)
388		- return
389		- }
390		-
391		- w.Header().Set("Content-Type", "application/json")
392		- type discoveryResponse struct {
393		- FeedURLs []string `json:"feed_urls"`
394		- Favicon string `json:"favicon"`
395		- }
396		- json.NewEncoder(w).Encode(discoveryResponse{
397		- FeedURLs: result.FeedURLs,
398		- Favicon: result.Favicon,
399		- })
400		-}
401		-
402	376	func (s Server) handleDismissFeedRecommendation(w http.ResponseWriter, r http.Request) {
403	377	user := currentUser(r)
404	378	feedURL := r.FormValue("feed_url")

  import (
  	"context"
  	"database/sql"
--	"encoding/json"
  	"errors"
  	"net/http"
  	"time"
  	})
  }
--func (s *Server) handleDiscoverFeedURL(w http.ResponseWriter, r *http.Request) {
--	siteURL := r.URL.Query().Get("url")
--	if siteURL == "" {
--		http.Error(w, "url required", http.StatusBadRequest)
--		return
--	}
--
--	result, err := feed.Discover(r.Context(), siteURL)
--	if err != nil {
--		s.logger.Error("feed discovery failed", "error", err, "url", siteURL)
--		http.Error(w, err.Error(), http.StatusInternalServerError)
--		return
--	}
--
--	w.Header().Set("Content-Type", "application/json")
--	type discoveryResponse struct {
--		FeedURLs []string `json:"feed_urls"`
--		Favicon  string   `json:"favicon"`
--	}
--	json.NewEncoder(w).Encode(discoveryResponse{
--		FeedURLs: result.FeedURLs,
--		Favicon:  result.Favicon,
--	})
--}
--
  func (s *Server) handleDismissFeedRecommendation(w http.ResponseWriter, r *http.Request) {
  	user := currentUser(r)
  	feedURL := r.FormValue("feed_url")

modified internal/server/server.go +0 -1 Viewed

@@ -161,7 +161,6 @@ func (s *Server) setupRoutes() {
161	161	r.Post("/refresh", s.handleRefreshFeeds)
162	162	r.Post("/retry", s.handleRetryFeed)
163	163	r.Get("/list", s.handleFeedList)
164		- r.Get("/discover-url", s.handleDiscoverFeedURL)
165	164	r.Post("/clear", s.handleClearAllSubscriptions)
166	165	r.Post("/dismiss", s.handleDismissFeedRecommendation)
167	166	})

  		r.Post("/refresh", s.handleRefreshFeeds)
  		r.Post("/retry", s.handleRetryFeed)
  		r.Get("/list", s.handleFeedList)
--		r.Get("/discover-url", s.handleDiscoverFeedURL)
  		r.Post("/clear", s.handleClearAllSubscriptions)
  		r.Post("/dismiss", s.handleDismissFeedRecommendation)
  	})

Improve fetcher performance and cleanup endpointsUnverified