nandi/glean · ee1eae0

nandi/gleanpublic⑂ Fork 0

⑂ ee1eae0

Refactor fetcher to include feed discovery logicUnverified

Julien Robert committed 2026-05-01T21:59:47+02:00 Browse files

ee1eae0 parent: aa5c138

modified internal/feed/fetcher.go +24 -0 Viewed

@@ -80,6 +80,30 @@ func (f Fetcher) Fetch(ctx context.Context, feedURL string) (ParseResult, erro
80	80	return nil, lastErr
81	81	}
82	82
	83	+func (f Fetcher) FetchOrDiscover(ctx context.Context, feedURL string) (ParseResult, string, error) {
	84	+ result, err := f.Fetch(ctx, feedURL)
	85	+ if err != nil && !strings.HasPrefix(feedURL, "at://") {
	86	+ return f.discover(ctx, feedURL)
	87	+ }
	88	+ return result, feedURL, err
	89	+}
	90	+
	91	+func (f Fetcher) discover(ctx context.Context, feedURL string) (ParseResult, string, error) {
	92	+ discovered, err := Discover(ctx, feedURL)
	93	+ if err != nil \|\| len(discovered.FeedURLs) == 0 {
	94	+ return nil, feedURL, fmt.Errorf("no feeds found at %s", feedURL)
	95	+ }
	96	+
	97	+ for _, candidate := range discovered.FeedURLs {
	98	+ result, fetchErr := f.Fetch(ctx, candidate)
	99	+ if fetchErr == nil && result != nil {
	100	+ return result, candidate, nil
	101	+ }
	102	+ }
	103	+
	104	+ return nil, feedURL, fmt.Errorf("no feeds found at %s", feedURL)
	105	+}
	106	+
83	107	func (f Fetcher) executeRequest(ctx context.Context, feedURL string) (ParseResult, *http.Response, error) {
84	108	req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
85	109	if err != nil {

  	return nil, lastErr
  }
++func (f *Fetcher) FetchOrDiscover(ctx context.Context, feedURL string) (*ParseResult, string, error) {
++	result, err := f.Fetch(ctx, feedURL)
++	if err != nil && !strings.HasPrefix(feedURL, "at://") {
++		return f.discover(ctx, feedURL)
++	}
++	return result, feedURL, err
++}
++
++func (f *Fetcher) discover(ctx context.Context, feedURL string) (*ParseResult, string, error) {
++	discovered, err := Discover(ctx, feedURL)
++	if err != nil || len(discovered.FeedURLs) == 0 {
++		return nil, feedURL, fmt.Errorf("no feeds found at %s", feedURL)
++	}
++
++	for _, candidate := range discovered.FeedURLs {
++		result, fetchErr := f.Fetch(ctx, candidate)
++		if fetchErr == nil && result != nil {
++			return result, candidate, nil
++		}
++	}
++
++	return nil, feedURL, fmt.Errorf("no feeds found at %s", feedURL)
++}
++
  func (f *Fetcher) executeRequest(ctx context.Context, feedURL string) (*ParseResult, *http.Response, error) {
  	req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
  	if err != nil {

modified internal/server/feeds_handler.go +1 -21 Viewed

@@ -4,7 +4,6 @@ import (
4	4	"context"
5	5	"database/sql"
6	6	"errors"
7		- "fmt"
8	7	"net/http"
9	8	"net/url"
10	9	"time"
@@ -132,10 +131,7 @@ func (s Server) handleAddFeed(w http.ResponseWriter, r http.Request) {
132	131	}
133	132	}
134	133
135		- result, err := s.fetcher.Fetch(r.Context(), feedURL)
136		- if err != nil && !atproto.IsATProtoFeedURL(feedURL) {
137		- result, feedURL, err = s.discoverFeed(r.Context(), feedURL)
138		- }
	134	+ result, feedURL, err := s.fetcher.FetchOrDiscover(r.Context(), feedURL)
139	135	if err != nil {
140	136	s.logger.Error("failed to fetch feed", "error", err, "url", feedURL)
141	137	http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -473,22 +469,6 @@ func (s Server) handleRetryFeed(w http.ResponseWriter, r http.Request) {
473	469	})
474	470	}
475	471
476		-func (s Server) discoverFeed(ctx context.Context, feedURL string) (feed.ParseResult, string, error) {
477		- discovered, err := feed.Discover(ctx, feedURL)
478		- if err != nil \|\| len(discovered.FeedURLs) == 0 {
479		- return nil, feedURL, fmt.Errorf("no feeds found at %s", feedURL)
480		- }
481		-
482		- for _, candidate := range discovered.FeedURLs {
483		- result, fetchErr := s.fetcher.Fetch(ctx, candidate)
484		- if fetchErr == nil && result != nil {
485		- return result, candidate, nil
486		- }
487		- }
488		-
489		- return nil, feedURL, fmt.Errorf("no feeds found at %s", feedURL)
490		-}
491		-
492	472	func nullString(s string) sql.NullString {
493	473	return sql.NullString{String: s, Valid: s != ""}
494	474	}

  	"context"
  	"database/sql"
  	"errors"
--	"fmt"
  	"net/http"
  	"net/url"
  	"time"
  		}
  	}
--	result, err := s.fetcher.Fetch(r.Context(), feedURL)
++	result, feedURL, err := s.fetcher.FetchOrDiscover(r.Context(), feedURL)
--	if err != nil && !atproto.IsATProtoFeedURL(feedURL) {
--		result, feedURL, err = s.discoverFeed(r.Context(), feedURL)
--	}
  	if err != nil {
  		s.logger.Error("failed to fetch feed", "error", err, "url", feedURL)
  		http.Error(w, err.Error(), http.StatusInternalServerError)
  	})
  }
--func (s *Server) discoverFeed(ctx context.Context, feedURL string) (*feed.ParseResult, string, error) {
--	discovered, err := feed.Discover(ctx, feedURL)
--	if err != nil || len(discovered.FeedURLs) == 0 {
--		return nil, feedURL, fmt.Errorf("no feeds found at %s", feedURL)
--	}
--
--	for _, candidate := range discovered.FeedURLs {
--		result, fetchErr := s.fetcher.Fetch(ctx, candidate)
--		if fetchErr == nil && result != nil {
--			return result, candidate, nil
--		}
--	}
--
--	return nil, feedURL, fmt.Errorf("no feeds found at %s", feedURL)
--}
--
  func nullString(s string) sql.NullString {
  	return sql.NullString{String: s, Valid: s != ""}
  }