rickub
nandi/gleanpublic⑂ Fork 0
⑂ 83ee19e
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Fix fetch feed when addedUnverified

Julien Robert committed 2026-04-28T11:47:00+02:00 Browse files
83ee19e parent: faac4b3
modified internal/server/feeds_handler.go +44 -26
@@ -78,21 +78,41 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
7878 }
7979
8080 s.render(w, r, "feeds.html", map[string]any{
81- "User": user,
82- "Subscriptions": subs,
83- "SubscriptionCount": subCount,
84- "Categories": categories,
85- "Category": category,
86- "FeedRecommendations": feedRecs,
87- "FollowedPeople": followedPeople,
88- "DiscoverPeople": discoverPeople,
89- "DeadFeeds": deadFeeds,
90- "Page": page,
91- "BaseURL": "/feeds",
92- "QueryParams": buildQueryParams(map[string]string{"category": category}),
81+ "User": user,
82+ "Subscriptions": subs,
83+ "SubscriptionCount": subCount,
84+ "Categories": categories,
85+ "Category": category,
86+ "FeedRecommendations": feedRecs,
87+ "FollowedPeople": followedPeople,
88+ "DiscoverPeople": discoverPeople,
89+ "DeadFeeds": deadFeeds,
90+ "Page": page,
91+ "BaseURL": "/feeds",
92+ "QueryParams": buildQueryParams(map[string]string{"category": category}),
9393 })
9494 }
9595
96+func (s *Server) storeFetchResult(ctx context.Context, feedURL, siteURL string, result *feed.ParseResult) {
97+ if result == nil {
98+ _ = s.dbs.Articles.MarkFeedFetched(ctx, feedURL)
99+ return
100+ }
101+ faviconURL := result.Feed.FaviconURL
102+ if faviconURL == "" {
103+ faviconURL = feed.ResolveFavicon(ctx, feedURL, siteURL)
104+ }
105+ if len(result.Articles) > 0 {
106+ if err := s.dbs.Articles.BatchUpsertArticles(ctx, result.Articles); err != nil {
107+ s.logger.Error("failed to store articles", "error", err, "feed", feedURL)
108+ }
109+ }
110+ _ = s.dbs.Articles.MarkFeedFetched(ctx, feedURL)
111+ if faviconURL != "" {
112+ _ = s.dbs.Articles.UpdateFeedFavicon(ctx, feedURL, faviconURL)
113+ }
114+}
115+
96116 func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
97117 user := currentUser(r)
98118 feedURL := r.FormValue("feed_url")
@@ -113,18 +133,10 @@ func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
113133 return
114134 }
115135
116- var feedTitle string
117- var faviconURL string
136+ var feedTitle, faviconURL string
118137 if result != nil {
119138 feedTitle = result.Feed.Title
120139 faviconURL = result.Feed.FaviconURL
121- if faviconURL == "" {
122- go func() {
123- if f := feed.ResolveFavicon(context.Background(), feedURL, result.Feed.SiteURL); f != "" {
124- _ = s.dbs.Articles.UpdateFeedFavicon(context.Background(), feedURL, f)
125- }
126- }()
127- }
128140 }
129141
130142 f := &db.Feed{
@@ -169,6 +181,8 @@ func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
169181 return
170182 }
171183
184+ go s.storeFetchResult(context.Background(), feedURL, result.Feed.SiteURL, result)
185+
172186 if err := s.engine.MarkImpressionActed(r.Context(), user.DID, "feed", feedURL); err != nil {
173187 s.logger.Warn("failed to mark impression acted", "error", err)
174188 }
@@ -272,7 +286,7 @@ func (s *Server) handleOPMLUpload(w http.ResponseWriter, r *http.Request) {
272286 var added int
273287 client := s.pdsClientForUser(r)
274288
275- var favGoroutines []struct{ feedURL, siteURL string }
289+ var feedsToFetch []struct{ feedURL, siteURL string }
276290 for _, fu := range feedURLs {
277291 f := &db.Feed{
278292 FeedURL: fu.URL,
@@ -285,7 +299,7 @@ func (s *Server) handleOPMLUpload(w http.ResponseWriter, r *http.Request) {
285299 continue
286300 }
287301
288- favGoroutines = append(favGoroutines, struct{ feedURL, siteURL string }{fu.URL, fu.SiteURL})
302+ feedsToFetch = append(feedsToFetch, struct{ feedURL, siteURL string }{fu.URL, fu.SiteURL})
289303
290304 var subURI, subCID string
291305 if client != nil {
@@ -316,11 +330,15 @@ func (s *Server) handleOPMLUpload(w http.ResponseWriter, r *http.Request) {
316330 go func() {
317331 g, ctx := errgroup.WithContext(context.Background())
318332 g.SetLimit(5)
319- for _, fav := range favGoroutines {
333+ for _, f := range feedsToFetch {
320334 g.Go(func() error {
321- if f := feed.ResolveFavicon(ctx, fav.feedURL, fav.siteURL); f != "" {
322- _ = s.dbs.Articles.UpdateFeedFavicon(ctx, fav.feedURL, f)
335+ result, err := s.fetcher.Fetch(ctx, f.feedURL)
336+ if err != nil {
337+ s.logger.Error("failed to fetch feed", "error", err, "feed", f.feedURL)
338+ _ = s.dbs.Articles.MarkFeedFetchError(ctx, f.feedURL, err.Error())
339+ return nil
323340 }
341+ s.storeFetchResult(ctx, f.feedURL, f.siteURL, result)
324342 return nil
325343 })
326344 }
@@ -78,21 +78,41 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
78 }78 }
79 79
80 s.render(w, r, "feeds.html", map[string]any{80 s.render(w, r, "feeds.html", map[string]any{
81- "User": user,81+ "User": user,
82- "Subscriptions": subs,82+ "Subscriptions": subs,
83- "SubscriptionCount": subCount,83+ "SubscriptionCount": subCount,
84- "Categories": categories,84+ "Categories": categories,
85- "Category": category,85+ "Category": category,
86- "FeedRecommendations": feedRecs,86+ "FeedRecommendations": feedRecs,
87- "FollowedPeople": followedPeople,87+ "FollowedPeople": followedPeople,
88- "DiscoverPeople": discoverPeople,88+ "DiscoverPeople": discoverPeople,
89- "DeadFeeds": deadFeeds,89+ "DeadFeeds": deadFeeds,
90- "Page": page,90+ "Page": page,
91- "BaseURL": "/feeds",91+ "BaseURL": "/feeds",
92- "QueryParams": buildQueryParams(map[string]string{"category": category}),92+ "QueryParams": buildQueryParams(map[string]string{"category": category}),
93 })93 })
94 }94 }
95 95
96+func (s *Server) storeFetchResult(ctx context.Context, feedURL, siteURL string, result *feed.ParseResult) {
97+ if result == nil {
98+ _ = s.dbs.Articles.MarkFeedFetched(ctx, feedURL)
99+ return
100+ }
101+ faviconURL := result.Feed.FaviconURL
102+ if faviconURL == "" {
103+ faviconURL = feed.ResolveFavicon(ctx, feedURL, siteURL)
104+ }
105+ if len(result.Articles) > 0 {
106+ if err := s.dbs.Articles.BatchUpsertArticles(ctx, result.Articles); err != nil {
107+ s.logger.Error("failed to store articles", "error", err, "feed", feedURL)
108+ }
109+ }
110+ _ = s.dbs.Articles.MarkFeedFetched(ctx, feedURL)
111+ if faviconURL != "" {
112+ _ = s.dbs.Articles.UpdateFeedFavicon(ctx, feedURL, faviconURL)
113+ }
114+}
115+
96 func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {116 func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
97 user := currentUser(r)117 user := currentUser(r)
98 feedURL := r.FormValue("feed_url")118 feedURL := r.FormValue("feed_url")
@@ -113,18 +133,10 @@ func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
113 return133 return
114 }134 }
115 135
116- var feedTitle string136+ var feedTitle, faviconURL string
117- var faviconURL string
118 if result != nil {137 if result != nil {
119 feedTitle = result.Feed.Title138 feedTitle = result.Feed.Title
120 faviconURL = result.Feed.FaviconURL139 faviconURL = result.Feed.FaviconURL
121- if faviconURL == "" {
122- go func() {
123- if f := feed.ResolveFavicon(context.Background(), feedURL, result.Feed.SiteURL); f != "" {
124- _ = s.dbs.Articles.UpdateFeedFavicon(context.Background(), feedURL, f)
125- }
126- }()
127- }
128 }140 }
129 141
130 f := &db.Feed{142 f := &db.Feed{
@@ -169,6 +181,8 @@ func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
169 return181 return
170 }182 }
171 183
184+ go s.storeFetchResult(context.Background(), feedURL, result.Feed.SiteURL, result)
185+
172 if err := s.engine.MarkImpressionActed(r.Context(), user.DID, "feed", feedURL); err != nil {186 if err := s.engine.MarkImpressionActed(r.Context(), user.DID, "feed", feedURL); err != nil {
173 s.logger.Warn("failed to mark impression acted", "error", err)187 s.logger.Warn("failed to mark impression acted", "error", err)
174 }188 }
@@ -272,7 +286,7 @@ func (s *Server) handleOPMLUpload(w http.ResponseWriter, r *http.Request) {
272 var added int286 var added int
273 client := s.pdsClientForUser(r)287 client := s.pdsClientForUser(r)
274 288
275- var favGoroutines []struct{ feedURL, siteURL string }289+ var feedsToFetch []struct{ feedURL, siteURL string }
276 for _, fu := range feedURLs {290 for _, fu := range feedURLs {
277 f := &db.Feed{291 f := &db.Feed{
278 FeedURL: fu.URL,292 FeedURL: fu.URL,
@@ -285,7 +299,7 @@ func (s *Server) handleOPMLUpload(w http.ResponseWriter, r *http.Request) {
285 continue299 continue
286 }300 }
287 301
288- favGoroutines = append(favGoroutines, struct{ feedURL, siteURL string }{fu.URL, fu.SiteURL})302+ feedsToFetch = append(feedsToFetch, struct{ feedURL, siteURL string }{fu.URL, fu.SiteURL})
289 303
290 var subURI, subCID string304 var subURI, subCID string
291 if client != nil {305 if client != nil {
@@ -316,11 +330,15 @@ func (s *Server) handleOPMLUpload(w http.ResponseWriter, r *http.Request) {
316 go func() {330 go func() {
317 g, ctx := errgroup.WithContext(context.Background())331 g, ctx := errgroup.WithContext(context.Background())
318 g.SetLimit(5)332 g.SetLimit(5)
319- for _, fav := range favGoroutines {333+ for _, f := range feedsToFetch {
320 g.Go(func() error {334 g.Go(func() error {
321- if f := feed.ResolveFavicon(ctx, fav.feedURL, fav.siteURL); f != "" {335+ result, err := s.fetcher.Fetch(ctx, f.feedURL)
322- _ = s.dbs.Articles.UpdateFeedFavicon(ctx, fav.feedURL, f)336+ if err != nil {
337+ s.logger.Error("failed to fetch feed", "error", err, "feed", f.feedURL)
338+ _ = s.dbs.Articles.MarkFeedFetchError(ctx, f.feedURL, err.Error())
339+ return nil
323 }340 }
341+ s.storeFetchResult(ctx, f.feedURL, f.siteURL, result)
324 return nil342 return nil
325 })343 })
326 }344 }