rickub
nandi/gleanpublic⑂ Fork 0
⑂ 238f0f3
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Make feed fetch scheduler configurable and improve update logicUnverified

Julien Robert committed 2026-04-23T09:49:08+02:00 Browse files
238f0f3 parent: 222591d
modified .env.example +3 -2
@@ -2,8 +2,9 @@ GLEAN_ADDR=:8080
22 GLEAN_DB=glean.db
33 GLEAN_JETSTREAM=wss://jetstream.glean.at
44 GLEAN_PLC_URL=https://didplc.glean.at
5-GLEAN_SYNC_INTERVAL=1h
6-GLEAN_CLUSTER_INTERVAL=10m
5+GLEAN_SYNC_INTERVAL=30m
6+GLEAN_CLUSTER_INTERVAL=1h
7+GLEAN_FETCH_INTERVAL=5m
78 GLEAN_COLLECTION_DIR_URL=https://lightrail.microcosm.blue/xrpc/com.atproto.sync.listReposByCollection?collection=at.glean.subscription
89 # Leave empty for localhost OAuth (development)
910 # GLEAN_OAUTH_CLIENT_ID=https://glean.at/oauth/client-metadata
@@ -2,8 +2,9 @@ GLEAN_ADDR=:8080
2 GLEAN_DB=glean.db2 GLEAN_DB=glean.db
3 GLEAN_JETSTREAM=wss://jetstream.glean.at3 GLEAN_JETSTREAM=wss://jetstream.glean.at
4 GLEAN_PLC_URL=https://didplc.glean.at4 GLEAN_PLC_URL=https://didplc.glean.at
5-GLEAN_SYNC_INTERVAL=1h5+GLEAN_SYNC_INTERVAL=30m
6-GLEAN_CLUSTER_INTERVAL=10m6+GLEAN_CLUSTER_INTERVAL=1h
7+GLEAN_FETCH_INTERVAL=5m
7 GLEAN_COLLECTION_DIR_URL=https://lightrail.microcosm.blue/xrpc/com.atproto.sync.listReposByCollection?collection=at.glean.subscription8 GLEAN_COLLECTION_DIR_URL=https://lightrail.microcosm.blue/xrpc/com.atproto.sync.listReposByCollection?collection=at.glean.subscription
8 # Leave empty for localhost OAuth (development)9 # Leave empty for localhost OAuth (development)
9 # GLEAN_OAUTH_CLIENT_ID=https://glean.at/oauth/client-metadata10 # GLEAN_OAUTH_CLIENT_ID=https://glean.at/oauth/client-metadata
modified docs/specs.md +2 -2
@@ -328,9 +328,9 @@ A background scheduler polls subscribed feeds on a fixed 5-minute tick. Feeds ar
328328
329329 ### 4.2 Fetch Schedule
330330
331-The scheduler uses a single fixed interval with in-flight deduplication:
331+The scheduler uses a configurable tick interval with in-flight deduplication:
332332
333-- **Tick interval**: The scheduler checks for stale feeds every 5 minutes
333+- **Tick interval**: The scheduler checks for stale feeds every `GLEAN_FETCH_INTERVAL` (default 5 minutes)
334334 - **Staleness threshold**: Feeds not fetched in the last 30 minutes are eligible
335335 - **Subscriber filter**: Only feeds with `subscriber_count > 0` are fetched
336336 - **In-flight dedup**: If a feed is already being fetched (e.g., manual refresh and background scheduler overlap), the second caller waits for the first to complete rather than fetching again
@@ -328,9 +328,9 @@ A background scheduler polls subscribed feeds on a fixed 5-minute tick. Feeds ar
328 328
329 ### 4.2 Fetch Schedule329 ### 4.2 Fetch Schedule
330 330
331-The scheduler uses a single fixed interval with in-flight deduplication:331+The scheduler uses a configurable tick interval with in-flight deduplication:
332 332
333-- **Tick interval**: The scheduler checks for stale feeds every 5 minutes333+- **Tick interval**: The scheduler checks for stale feeds every `GLEAN_FETCH_INTERVAL` (default 5 minutes)
334 - **Staleness threshold**: Feeds not fetched in the last 30 minutes are eligible334 - **Staleness threshold**: Feeds not fetched in the last 30 minutes are eligible
335 - **Subscriber filter**: Only feeds with `subscriber_count > 0` are fetched335 - **Subscriber filter**: Only feeds with `subscriber_count > 0` are fetched
336 - **In-flight dedup**: If a feed is already being fetched (e.g., manual refresh and background scheduler overlap), the second caller waits for the first to complete rather than fetching again336 - **In-flight dedup**: If a feed is already being fetched (e.g., manual refresh and background scheduler overlap), the second caller waits for the first to complete rather than fetching again
modified internal/atproto/jetstream.go +1 -1
@@ -130,7 +130,7 @@ func (jc *JetstreamConsumer) Start(ctx context.Context) error {
130130 return ctx.Err()
131131 }
132132 if err != nil {
133- jc.logger.Error("jetstream connection error", "error", err)
133+ jc.logger.Warn("jetstream connection lost", "error", err)
134134 metrics.JetstreamReconnects.Inc()
135135 }
136136
@@ -130,7 +130,7 @@ func (jc *JetstreamConsumer) Start(ctx context.Context) error {
130 return ctx.Err()130 return ctx.Err()
131 }131 }
132 if err != nil {132 if err != nil {
133- jc.logger.Error("jetstream connection error", "error", err)133+ jc.logger.Warn("jetstream connection lost", "error", err)
134 metrics.JetstreamReconnects.Inc()134 metrics.JetstreamReconnects.Inc()
135 }135 }
136 136
modified internal/cluster/social.go +8 -6
@@ -17,12 +17,14 @@ func (e *Engine) ComputeFollowDistances(ctx context.Context) error {
1717
1818 _, err = tx.ExecContext(ctx, `
1919 INSERT INTO follow_distances (user_a, user_b, distance)
20- SELECT user_did, target_did, 1 FROM follows WHERE user_did != target_did
21- UNION ALL
22- SELECT f1.user_did, f2.target_did, 2
23- FROM follows f1
24- JOIN follows f2 ON f1.target_did = f2.user_did
25- WHERE f1.user_did != f2.target_did
20+ SELECT user_a, user_b, MIN(distance) FROM (
21+ SELECT user_did AS user_a, target_did AS user_b, 1 AS distance FROM follows WHERE user_did != target_did
22+ UNION ALL
23+ SELECT f1.user_did, f2.target_did, 2
24+ FROM follows f1
25+ JOIN follows f2 ON f1.target_did = f2.user_did
26+ WHERE f1.user_did != f2.target_did
27+ ) GROUP BY user_a, user_b
2628 `)
2729 if err != nil {
2830 return err
@@ -17,12 +17,14 @@ func (e *Engine) ComputeFollowDistances(ctx context.Context) error {
17 17
18 _, err = tx.ExecContext(ctx, `18 _, err = tx.ExecContext(ctx, `
19 INSERT INTO follow_distances (user_a, user_b, distance)19 INSERT INTO follow_distances (user_a, user_b, distance)
20- SELECT user_did, target_did, 1 FROM follows WHERE user_did != target_did20+ SELECT user_a, user_b, MIN(distance) FROM (
21- UNION ALL21+ SELECT user_did AS user_a, target_did AS user_b, 1 AS distance FROM follows WHERE user_did != target_did
22- SELECT f1.user_did, f2.target_did, 222+ UNION ALL
23- FROM follows f123+ SELECT f1.user_did, f2.target_did, 2
24- JOIN follows f2 ON f1.target_did = f2.user_did24+ FROM follows f1
25- WHERE f1.user_did != f2.target_did25+ JOIN follows f2 ON f1.target_did = f2.user_did
26+ WHERE f1.user_did != f2.target_did
27+ ) GROUP BY user_a, user_b
26 `)28 `)
27 if err != nil {29 if err != nil {
28 return err30 return err
modified internal/db/article.go +0 -25
@@ -37,36 +37,11 @@ type ReadState struct {
3737 ReadAt sql.NullTime
3838 }
3939
40-func (db *DB) UpsertArticle(ctx context.Context, article *Article) (int64, error) {
41- var id int64
42- err := db.QueryRowContext(ctx, `
43- INSERT INTO articles (feed_url, guid, title, url, author, summary, content, published, updated)
44- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
45- ON CONFLICT(feed_url, guid) DO NOTHING
46- RETURNING id
47- `, article.FeedURL, article.GUID, article.Title, article.URL, article.Author,
48- article.Summary, article.Content, article.Published, article.Updated).Scan(&id)
49- if err == sql.ErrNoRows {
50- err = db.QueryRowContext(ctx, `
51- SELECT id FROM articles WHERE feed_url = ? AND guid = ?
52- `, article.FeedURL, article.GUID).Scan(&id)
53- }
54- return id, err
55-}
56-
5740 func (db *DB) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
5841 if len(articles) == 0 {
5942 return nil
6043 }
6144
62- err := upsertArticlesBatch(ctx, db, articles)
63- if err != nil {
64- err = upsertArticlesBatch(ctx, db, articles)
65- }
66- return err
67-}
68-
69-func upsertArticlesBatch(ctx context.Context, db *DB, articles []feed.Article) error {
7045 tx, err := db.BeginTx(ctx, nil)
7146 if err != nil {
7247 return err
@@ -37,36 +37,11 @@ type ReadState struct {
37 ReadAt sql.NullTime37 ReadAt sql.NullTime
38 }38 }
39 39
40-func (db *DB) UpsertArticle(ctx context.Context, article *Article) (int64, error) {
41- var id int64
42- err := db.QueryRowContext(ctx, `
43- INSERT INTO articles (feed_url, guid, title, url, author, summary, content, published, updated)
44- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
45- ON CONFLICT(feed_url, guid) DO NOTHING
46- RETURNING id
47- `, article.FeedURL, article.GUID, article.Title, article.URL, article.Author,
48- article.Summary, article.Content, article.Published, article.Updated).Scan(&id)
49- if err == sql.ErrNoRows {
50- err = db.QueryRowContext(ctx, `
51- SELECT id FROM articles WHERE feed_url = ? AND guid = ?
52- `, article.FeedURL, article.GUID).Scan(&id)
53- }
54- return id, err
55-}
56-
57 func (db *DB) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {40 func (db *DB) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
58 if len(articles) == 0 {41 if len(articles) == 0 {
59 return nil42 return nil
60 }43 }
61 44
62- err := upsertArticlesBatch(ctx, db, articles)
63- if err != nil {
64- err = upsertArticlesBatch(ctx, db, articles)
65- }
66- return err
67-}
68-
69-func upsertArticlesBatch(ctx context.Context, db *DB, articles []feed.Article) error {
70 tx, err := db.BeginTx(ctx, nil)45 tx, err := db.BeginTx(ctx, nil)
71 if err != nil {46 if err != nil {
72 return err47 return err
modified internal/db/store.go +16 -31
@@ -2,7 +2,6 @@ package db
22
33 import (
44 "context"
5- "database/sql"
65 "time"
76
87 "pkg.rbrt.fr/glean/internal/feed"
@@ -37,37 +36,23 @@ func (a *FeedStoreAdapter) GetFeedsToFetch(ctx context.Context, olderThan time.D
3736 return feeds, nil
3837 }
3938
40-func (a *FeedStoreAdapter) UpsertArticle(ctx context.Context, article *feed.Article) (int64, error) {
41- dbArticle := &Article{
42- FeedURL: article.FeedURL,
43- GUID: article.GUID,
44- Title: article.Title,
45- Summary: sql.NullString{String: article.Summary, Valid: article.Summary != ""},
46- Content: sql.NullString{String: article.Content, Valid: article.Content != ""},
47- Author: sql.NullString{String: article.Author, Valid: article.Author != ""},
48- URL: sql.NullString{String: article.URL, Valid: article.URL != ""},
49- }
50- if !article.Published.IsZero() {
51- dbArticle.Published = sql.NullTime{Time: article.Published, Valid: true}
52- }
53- if !article.Updated.IsZero() {
54- dbArticle.Updated = sql.NullTime{Time: article.Updated, Valid: true}
55- }
56- return a.db.UpsertArticle(ctx, dbArticle)
57-}
58-
59-func (a *FeedStoreAdapter) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
60- return a.db.UpsertArticlesBatch(ctx, articles)
61-}
62-
63-func (a *FeedStoreAdapter) MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error {
64- return a.db.MarkFeedFetched(ctx, feedURL, etag, lastModified)
65-}
66-
67-func (a *FeedStoreAdapter) MarkFeedFetchError(ctx context.Context, feedURL, lastError string) error {
39+func (a *FeedStoreAdapter) RecordFetchError(ctx context.Context, feedURL, lastError string) error {
6840 return a.db.MarkFeedFetchError(ctx, feedURL, lastError)
6941 }
7042
71-func (a *FeedStoreAdapter) UpdateFeedFavicon(ctx context.Context, feedURL, faviconURL string) error {
72- return a.db.UpdateFeedFavicon(ctx, feedURL, faviconURL)
43+func (a *FeedStoreAdapter) StoreFetchResult(ctx context.Context, feedURL, etag, lastModified string, articles []feed.Article, faviconURL string) error {
44+ if err := a.db.MarkFeedFetched(ctx, feedURL, etag, lastModified); err != nil {
45+ return err
46+ }
47+ if len(articles) > 0 {
48+ if err := a.db.UpsertArticlesBatch(ctx, articles); err != nil {
49+ return err
50+ }
51+ }
52+ if faviconURL != "" {
53+ if err := a.db.UpdateFeedFavicon(ctx, feedURL, faviconURL); err != nil {
54+ return err
55+ }
56+ }
57+ return nil
7358 }
@@ -2,7 +2,6 @@ package db
2 2
3 import (3 import (
4 "context"4 "context"
5- "database/sql"
6 "time"5 "time"
7 6
8 "pkg.rbrt.fr/glean/internal/feed"7 "pkg.rbrt.fr/glean/internal/feed"
@@ -37,37 +36,23 @@ func (a *FeedStoreAdapter) GetFeedsToFetch(ctx context.Context, olderThan time.D
37 return feeds, nil36 return feeds, nil
38 }37 }
39 38
40-func (a *FeedStoreAdapter) UpsertArticle(ctx context.Context, article *feed.Article) (int64, error) {39+func (a *FeedStoreAdapter) RecordFetchError(ctx context.Context, feedURL, lastError string) error {
41- dbArticle := &Article{
42- FeedURL: article.FeedURL,
43- GUID: article.GUID,
44- Title: article.Title,
45- Summary: sql.NullString{String: article.Summary, Valid: article.Summary != ""},
46- Content: sql.NullString{String: article.Content, Valid: article.Content != ""},
47- Author: sql.NullString{String: article.Author, Valid: article.Author != ""},
48- URL: sql.NullString{String: article.URL, Valid: article.URL != ""},
49- }
50- if !article.Published.IsZero() {
51- dbArticle.Published = sql.NullTime{Time: article.Published, Valid: true}
52- }
53- if !article.Updated.IsZero() {
54- dbArticle.Updated = sql.NullTime{Time: article.Updated, Valid: true}
55- }
56- return a.db.UpsertArticle(ctx, dbArticle)
57-}
58-
59-func (a *FeedStoreAdapter) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
60- return a.db.UpsertArticlesBatch(ctx, articles)
61-}
62-
63-func (a *FeedStoreAdapter) MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error {
64- return a.db.MarkFeedFetched(ctx, feedURL, etag, lastModified)
65-}
66-
67-func (a *FeedStoreAdapter) MarkFeedFetchError(ctx context.Context, feedURL, lastError string) error {
68 return a.db.MarkFeedFetchError(ctx, feedURL, lastError)40 return a.db.MarkFeedFetchError(ctx, feedURL, lastError)
69 }41 }
70 42
71-func (a *FeedStoreAdapter) UpdateFeedFavicon(ctx context.Context, feedURL, faviconURL string) error {43+func (a *FeedStoreAdapter) StoreFetchResult(ctx context.Context, feedURL, etag, lastModified string, articles []feed.Article, faviconURL string) error {
72- return a.db.UpdateFeedFavicon(ctx, feedURL, faviconURL)44+ if err := a.db.MarkFeedFetched(ctx, feedURL, etag, lastModified); err != nil {
45+ return err
46+ }
47+ if len(articles) > 0 {
48+ if err := a.db.UpsertArticlesBatch(ctx, articles); err != nil {
49+ return err
50+ }
51+ }
52+ if faviconURL != "" {
53+ if err := a.db.UpdateFeedFavicon(ctx, feedURL, faviconURL); err != nil {
54+ return err
55+ }
56+ }
57+ return nil
73 }58 }
modified internal/feed/discover.go +3 -6
@@ -63,10 +63,7 @@ func cleanFavicon(s string) string {
6363 return strings.TrimRight(s, "/")
6464 }
6565
66-func ResolveFavicon(ctx context.Context, feedURL, siteURL, parsedFavicon string) string {
67- if parsedFavicon != "" {
68- return cleanFavicon(parsedFavicon)
69- }
66+func ResolveFavicon(ctx context.Context, feedURL, siteURL string) string {
7067 target := siteURL
7168 if target == "" {
7269 target = feedURL
@@ -117,7 +114,7 @@ func findFavicon(ctx context.Context, base *url.URL, links []string) string {
117114 origin.Fragment = ""
118115
119116 type result struct {
120- url string
117+ url string
121118 found bool
122119 }
123120 found := make(chan result, 1)
@@ -201,4 +198,4 @@ func fetchHTML(ctx context.Context, siteURL string) (*url.URL, string) {
201198
202199 base := resp.Request.URL
203200 return base, string(body)
204-}
\ No newline at end of file
201+}
@@ -63,10 +63,7 @@ func cleanFavicon(s string) string {
63 return strings.TrimRight(s, "/")63 return strings.TrimRight(s, "/")
64 }64 }
65 65
66-func ResolveFavicon(ctx context.Context, feedURL, siteURL, parsedFavicon string) string {66+func ResolveFavicon(ctx context.Context, feedURL, siteURL string) string {
67- if parsedFavicon != "" {
68- return cleanFavicon(parsedFavicon)
69- }
70 target := siteURL67 target := siteURL
71 if target == "" {68 if target == "" {
72 target = feedURL69 target = feedURL
@@ -117,7 +114,7 @@ func findFavicon(ctx context.Context, base *url.URL, links []string) string {
117 origin.Fragment = ""114 origin.Fragment = ""
118 115
119 type result struct {116 type result struct {
120- url string117+ url string
121 found bool118 found bool
122 }119 }
123 found := make(chan result, 1)120 found := make(chan result, 1)
@@ -201,4 +198,4 @@ func fetchHTML(ctx context.Context, siteURL string) (*url.URL, string) {
201 198
202 base := resp.Request.URL199 base := resp.Request.URL
203 return base, string(body)200 return base, string(body)
204-}
\ No newline at end of file\ No newline at end of file
201+}
modified internal/feed/fetcher.go +34 -48
@@ -69,11 +69,8 @@ func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string)
6969
7070 type FeedStore interface {
7171 GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error)
72- UpsertArticle(ctx context.Context, article *Article) (int64, error)
73- UpsertArticlesBatch(ctx context.Context, articles []Article) error
74- MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error
75- MarkFeedFetchError(ctx context.Context, feedURL, lastError string) error
76- UpdateFeedFavicon(ctx context.Context, feedURL, faviconURL string) error
72+ StoreFetchResult(ctx context.Context, feedURL, etag, lastModified string, articles []Article, faviconURL string) error
73+ RecordFetchError(ctx context.Context, feedURL, lastError string) error
7774 }
7875
7976 type fetchCall struct {
@@ -81,29 +78,29 @@ type fetchCall struct {
8178 }
8279
8380 type Scheduler struct {
84- fetcher *Fetcher
85- store FeedStore
86- logger *slog.Logger
87- interval time.Duration
88- inFlight sync.Map
81+ fetcher *Fetcher
82+ store FeedStore
83+ logger *slog.Logger
84+ tickInterval time.Duration
85+ staleInterval time.Duration
86+ inFlight sync.Map
8987 }
9088
91-func NewScheduler(store FeedStore, logger *slog.Logger) *Scheduler {
89+func NewScheduler(store FeedStore, logger *slog.Logger, tickInterval, staleInterval time.Duration) *Scheduler {
9290 return &Scheduler{
93- fetcher: NewFetcher(),
94- store: store,
95- logger: logger,
96- interval: 30 * time.Minute,
97- inFlight: sync.Map{},
91+ fetcher: NewFetcher(),
92+ store: store,
93+ logger: logger,
94+ tickInterval: tickInterval,
95+ staleInterval: staleInterval,
96+ inFlight: sync.Map{},
9897 }
9998 }
10099
101100 func (s *Scheduler) Run(ctx context.Context) error {
102- ticker := time.NewTicker(5 * time.Minute)
101+ ticker := time.NewTicker(s.tickInterval)
103102 defer ticker.Stop()
104103
105- s.fetchAll(ctx)
106-
107104 for {
108105 select {
109106 case <-ctx.Done():
@@ -115,7 +112,7 @@ func (s *Scheduler) Run(ctx context.Context) error {
115112 }
116113
117114 func (s *Scheduler) fetchAll(ctx context.Context) {
118- feeds, err := s.store.GetFeedsToFetch(ctx, s.interval, 200)
115+ feeds, err := s.store.GetFeedsToFetch(ctx, s.staleInterval, 500)
119116 if err != nil {
120117 s.logger.Error("failed to get feeds", "error", err)
121118 return
@@ -151,45 +148,34 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) {
151148 start := time.Now()
152149 result, newEtag, newLastModified, err := s.fetcher.Fetch(ctx, feed.URL, feed.ETag, feed.LastModified)
153150 metrics.FeedsFetchedDuration.Observe(time.Since(start).Seconds())
151+ metrics.FeedsFetched.Inc()
152+ metrics.FeedsFetchedLast.Set(float64(time.Now().Unix()))
154153 if err != nil {
155- metrics.FeedsFetched.WithLabelValues("error").Inc()
156154 s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL)
157- if updErr := s.store.MarkFeedFetchError(ctx, feed.URL, err.Error()); updErr != nil {
158- s.logger.Error("failed to update feed fetch error", "error", updErr, "feed", feed.URL)
159- }
155+ s.store.RecordFetchError(ctx, feed.URL, err.Error())
160156 return
161157 }
162158
163159 if result == nil {
164- metrics.FeedsFetched.WithLabelValues("not_modified").Inc()
165- if updErr := s.store.MarkFeedFetched(ctx, feed.URL, feed.ETag, feed.LastModified); updErr != nil {
166- s.logger.Error("failed to update feed fetch result", "error", updErr, "feed", feed.URL)
160+ s.logger.Info("fetched articles", "feed", feed.URL, "count", 0)
161+ if err := s.store.StoreFetchResult(ctx, feed.URL, newEtag, newLastModified, nil, ""); err != nil {
162+ s.logger.Error("failed to store feed fetch result", "error", err, "feed", feed.URL)
167163 }
168164 return
169165 }
170166
171- metrics.FeedsFetched.WithLabelValues("success").Inc()
172-
173- for _, article := range result.Articles {
174- article.FeedURL = feed.URL
175- }
176- if err := s.store.UpsertArticlesBatch(ctx, result.Articles); err != nil {
177- s.logger.Error("failed to upsert articles", "error", err, "feed", feed.URL)
178- } else {
179- metrics.ArticlesUpserted.Add(float64(len(result.Articles)))
167+ faviconURL := result.Feed.FaviconURL
168+ if faviconURL == "" && feed.FaviconURL == "" {
169+ faviconURL = ResolveFavicon(context.Background(), feed.URL, feed.SiteURL)
180170 }
181171
182- if err := s.store.MarkFeedFetched(ctx, feed.URL, newEtag, newLastModified); err != nil {
183- s.logger.Error("failed to update feed fetch result", "error", err, "feed", feed.URL)
184- }
185-
186- if result != nil && result.Feed.FaviconURL != "" {
187- _ = s.store.UpdateFeedFavicon(ctx, feed.URL, result.Feed.FaviconURL)
188- } else if feed.FaviconURL == "" {
189- go func() {
190- if f := ResolveFavicon(context.Background(), feed.URL, feed.SiteURL, ""); f != "" {
191- _ = s.store.UpdateFeedFavicon(context.Background(), feed.URL, f)
192- }
193- }()
172+ if err := s.store.StoreFetchResult(ctx, feed.URL, newEtag, newLastModified, result.Articles, faviconURL); err != nil {
173+ s.logger.Error("failed to store feed fetch result", "error", err, "feed", feed.URL)
174+ } else {
175+ articleCount := len(result.Articles)
176+ s.logger.Info("fetched articles", "feed", feed.URL, "count", articleCount)
177+ if articleCount > 0 {
178+ metrics.ArticlesUpserted.Add(float64(articleCount))
179+ }
194180 }
195181 }
@@ -69,11 +69,8 @@ func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string)
69 69
70 type FeedStore interface {70 type FeedStore interface {
71 GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error)71 GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error)
72- UpsertArticle(ctx context.Context, article *Article) (int64, error)72+ StoreFetchResult(ctx context.Context, feedURL, etag, lastModified string, articles []Article, faviconURL string) error
73- UpsertArticlesBatch(ctx context.Context, articles []Article) error73+ RecordFetchError(ctx context.Context, feedURL, lastError string) error
74- MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error
75- MarkFeedFetchError(ctx context.Context, feedURL, lastError string) error
76- UpdateFeedFavicon(ctx context.Context, feedURL, faviconURL string) error
77 }74 }
78 75
79 type fetchCall struct {76 type fetchCall struct {
@@ -81,29 +78,29 @@ type fetchCall struct {
81 }78 }
82 79
83 type Scheduler struct {80 type Scheduler struct {
84- fetcher *Fetcher81+ fetcher *Fetcher
85- store FeedStore82+ store FeedStore
86- logger *slog.Logger83+ logger *slog.Logger
87- interval time.Duration84+ tickInterval time.Duration
88- inFlight sync.Map85+ staleInterval time.Duration
86+ inFlight sync.Map
89 }87 }
90 88
91-func NewScheduler(store FeedStore, logger *slog.Logger) *Scheduler {89+func NewScheduler(store FeedStore, logger *slog.Logger, tickInterval, staleInterval time.Duration) *Scheduler {
92 return &Scheduler{90 return &Scheduler{
93- fetcher: NewFetcher(),91+ fetcher: NewFetcher(),
94- store: store,92+ store: store,
95- logger: logger,93+ logger: logger,
96- interval: 30 * time.Minute,94+ tickInterval: tickInterval,
97- inFlight: sync.Map{},95+ staleInterval: staleInterval,
96+ inFlight: sync.Map{},
98 }97 }
99 }98 }
100 99
101 func (s *Scheduler) Run(ctx context.Context) error {100 func (s *Scheduler) Run(ctx context.Context) error {
102- ticker := time.NewTicker(5 * time.Minute)101+ ticker := time.NewTicker(s.tickInterval)
103 defer ticker.Stop()102 defer ticker.Stop()
104 103
105- s.fetchAll(ctx)
106-
107 for {104 for {
108 select {105 select {
109 case <-ctx.Done():106 case <-ctx.Done():
@@ -115,7 +112,7 @@ func (s *Scheduler) Run(ctx context.Context) error {
115 }112 }
116 113
117 func (s *Scheduler) fetchAll(ctx context.Context) {114 func (s *Scheduler) fetchAll(ctx context.Context) {
118- feeds, err := s.store.GetFeedsToFetch(ctx, s.interval, 200)115+ feeds, err := s.store.GetFeedsToFetch(ctx, s.staleInterval, 500)
119 if err != nil {116 if err != nil {
120 s.logger.Error("failed to get feeds", "error", err)117 s.logger.Error("failed to get feeds", "error", err)
121 return118 return
@@ -151,45 +148,34 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) {
151 start := time.Now()148 start := time.Now()
152 result, newEtag, newLastModified, err := s.fetcher.Fetch(ctx, feed.URL, feed.ETag, feed.LastModified)149 result, newEtag, newLastModified, err := s.fetcher.Fetch(ctx, feed.URL, feed.ETag, feed.LastModified)
153 metrics.FeedsFetchedDuration.Observe(time.Since(start).Seconds())150 metrics.FeedsFetchedDuration.Observe(time.Since(start).Seconds())
151+ metrics.FeedsFetched.Inc()
152+ metrics.FeedsFetchedLast.Set(float64(time.Now().Unix()))
154 if err != nil {153 if err != nil {
155- metrics.FeedsFetched.WithLabelValues("error").Inc()
156 s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL)154 s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL)
157- if updErr := s.store.MarkFeedFetchError(ctx, feed.URL, err.Error()); updErr != nil {155+ s.store.RecordFetchError(ctx, feed.URL, err.Error())
158- s.logger.Error("failed to update feed fetch error", "error", updErr, "feed", feed.URL)
159- }
160 return156 return
161 }157 }
162 158
163 if result == nil {159 if result == nil {
164- metrics.FeedsFetched.WithLabelValues("not_modified").Inc()160+ s.logger.Info("fetched articles", "feed", feed.URL, "count", 0)
165- if updErr := s.store.MarkFeedFetched(ctx, feed.URL, feed.ETag, feed.LastModified); updErr != nil {161+ if err := s.store.StoreFetchResult(ctx, feed.URL, newEtag, newLastModified, nil, ""); err != nil {
166- s.logger.Error("failed to update feed fetch result", "error", updErr, "feed", feed.URL)162+ s.logger.Error("failed to store feed fetch result", "error", err, "feed", feed.URL)
167 }163 }
168 return164 return
169 }165 }
170 166
171- metrics.FeedsFetched.WithLabelValues("success").Inc()167+ faviconURL := result.Feed.FaviconURL
172-168+ if faviconURL == "" && feed.FaviconURL == "" {
173- for _, article := range result.Articles {169+ faviconURL = ResolveFavicon(context.Background(), feed.URL, feed.SiteURL)
174- article.FeedURL = feed.URL
175- }
176- if err := s.store.UpsertArticlesBatch(ctx, result.Articles); err != nil {
177- s.logger.Error("failed to upsert articles", "error", err, "feed", feed.URL)
178- } else {
179- metrics.ArticlesUpserted.Add(float64(len(result.Articles)))
180 }170 }
181 171
182- if err := s.store.MarkFeedFetched(ctx, feed.URL, newEtag, newLastModified); err != nil {172+ if err := s.store.StoreFetchResult(ctx, feed.URL, newEtag, newLastModified, result.Articles, faviconURL); err != nil {
183- s.logger.Error("failed to update feed fetch result", "error", err, "feed", feed.URL)173+ s.logger.Error("failed to store feed fetch result", "error", err, "feed", feed.URL)
184- }174+ } else {
185-175+ articleCount := len(result.Articles)
186- if result != nil && result.Feed.FaviconURL != "" {176+ s.logger.Info("fetched articles", "feed", feed.URL, "count", articleCount)
187- _ = s.store.UpdateFeedFavicon(ctx, feed.URL, result.Feed.FaviconURL)177+ if articleCount > 0 {
188- } else if feed.FaviconURL == "" {178+ metrics.ArticlesUpserted.Add(float64(articleCount))
189- go func() {179+ }
190- if f := ResolveFavicon(context.Background(), feed.URL, feed.SiteURL, ""); f != "" {
191- _ = s.store.UpdateFeedFavicon(context.Background(), feed.URL, f)
192- }
193- }()
194 }180 }
195 }181 }
modified internal/feed/opml.go +2 -10
@@ -4,6 +4,7 @@ import (
44 "bytes"
55 "encoding/xml"
66 "io"
7+ "slices"
78 "strings"
89 )
910
@@ -115,7 +116,7 @@ func GenerateOPML(feeds []FeedURL, title string) ([]byte, error) {
115116 }
116117 if f.Category != "" {
117118 categoryMap[f.Category] = append(categoryMap[f.Category], outline)
118- if !contains(categories, f.Category) {
119+ if !slices.Contains(categories, f.Category) {
119120 categories = append(categories, f.Category)
120121 }
121122 } else {
@@ -143,12 +144,3 @@ func GenerateOPML(feeds []FeedURL, title string) ([]byte, error) {
143144
144145 return buf.Bytes(), nil
145146 }
146-
147-func contains(slice []string, s string) bool {
148- for _, v := range slice {
149- if v == s {
150- return true
151- }
152- }
153- return false
154-}
@@ -4,6 +4,7 @@ import (
4 "bytes"4 "bytes"
5 "encoding/xml"5 "encoding/xml"
6 "io"6 "io"
7+ "slices"
7 "strings"8 "strings"
8 )9 )
9 10
@@ -115,7 +116,7 @@ func GenerateOPML(feeds []FeedURL, title string) ([]byte, error) {
115 }116 }
116 if f.Category != "" {117 if f.Category != "" {
117 categoryMap[f.Category] = append(categoryMap[f.Category], outline)118 categoryMap[f.Category] = append(categoryMap[f.Category], outline)
118- if !contains(categories, f.Category) {119+ if !slices.Contains(categories, f.Category) {
119 categories = append(categories, f.Category)120 categories = append(categories, f.Category)
120 }121 }
121 } else {122 } else {
@@ -143,12 +144,3 @@ func GenerateOPML(feeds []FeedURL, title string) ([]byte, error) {
143 144
144 return buf.Bytes(), nil145 return buf.Bytes(), nil
145 }146 }
146-
147-func contains(slice []string, s string) bool {
148- for _, v := range slice {
149- if v == s {
150- return true
151- }
152- }
153- return false
154-}
modified internal/metrics/metrics.go +7 -2
@@ -6,10 +6,15 @@ import (
66 )
77
88 var (
9- FeedsFetched = promauto.NewCounterVec(prometheus.CounterOpts{
9+ FeedsFetched = promauto.NewCounter(prometheus.CounterOpts{
1010 Name: "glean_feeds_fetched_total",
1111 Help: "Total number of feed fetch attempts",
12- }, []string{"status"})
12+ })
13+
14+ FeedsFetchedLast = promauto.NewGauge(prometheus.GaugeOpts{
15+ Name: "glean_feeds_fetched_last_timestamp_seconds",
16+ Help: "Unix timestamp of last feed fetch",
17+ })
1318
1419 FeedsFetchedDuration = promauto.NewHistogram(prometheus.HistogramOpts{
1520 Name: "glean_feed_fetch_duration_seconds",
@@ -6,10 +6,15 @@ import (
6 )6 )
7 7
8 var (8 var (
9- FeedsFetched = promauto.NewCounterVec(prometheus.CounterOpts{9+ FeedsFetched = promauto.NewCounter(prometheus.CounterOpts{
10 Name: "glean_feeds_fetched_total",10 Name: "glean_feeds_fetched_total",
11 Help: "Total number of feed fetch attempts",11 Help: "Total number of feed fetch attempts",
12- }, []string{"status"})12+ })
13+
14+ FeedsFetchedLast = promauto.NewGauge(prometheus.GaugeOpts{
15+ Name: "glean_feeds_fetched_last_timestamp_seconds",
16+ Help: "Unix timestamp of last feed fetch",
17+ })
13 18
14 FeedsFetchedDuration = promauto.NewHistogram(prometheus.HistogramOpts{19 FeedsFetchedDuration = promauto.NewHistogram(prometheus.HistogramOpts{
15 Name: "glean_feed_fetch_duration_seconds",20 Name: "glean_feed_fetch_duration_seconds",
modified internal/server/feeds_handler.go +24 -22
@@ -72,30 +72,32 @@ func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
7272 return
7373 }
7474
75+ var feedTitle string
76+ var faviconURL string
7577 if result != nil {
76- f := &db.Feed{
77- FeedURL: feedURL,
78- Title: nullString(result.Feed.Title),
79- SiteURL: nullString(result.Feed.SiteURL),
80- Description: nullString(result.Feed.Description),
81- FeedType: nullString(result.Feed.Type),
82- }
83- if err := s.db.UpsertFeed(r.Context(), f); err != nil {
84- s.logger.Error("failed to upsert feed", "error", err)
85- http.Error(w, err.Error(), http.StatusInternalServerError)
86- return
78+ feedTitle = result.Feed.Title
79+ faviconURL = result.Feed.FaviconURL
80+ if faviconURL == "" {
81+ go func() {
82+ if f := feed.ResolveFavicon(context.Background(), feedURL, result.Feed.SiteURL); f != "" {
83+ _ = s.db.UpdateFeedFavicon(context.Background(), feedURL, f)
84+ }
85+ }()
8786 }
88-
89- go func() {
90- if f := feed.ResolveFavicon(context.Background(), feedURL, result.Feed.SiteURL, result.Feed.FaviconURL); f != "" {
91- _ = s.db.UpdateFeedFavicon(context.Background(), feedURL, f)
92- }
93- }()
9487 }
9588
96- var feedTitle string
97- if result != nil {
98- feedTitle = result.Feed.Title
89+ f := &db.Feed{
90+ FeedURL: feedURL,
91+ Title: nullString(feedTitle),
92+ SiteURL: nullString(result.Feed.SiteURL),
93+ Description: nullString(result.Feed.Description),
94+ FeedType: nullString(result.Feed.Type),
95+ FaviconURL: nullString(faviconURL),
96+ }
97+ if err := s.db.UpsertFeed(r.Context(), f); err != nil {
98+ s.logger.Error("failed to upsert feed", "error", err)
99+ http.Error(w, err.Error(), http.StatusInternalServerError)
100+ return
99101 }
100102
101103 var subURI, subCID string
@@ -235,8 +237,8 @@ func (s *Server) handleOPMLUpload(w http.ResponseWriter, r *http.Request) {
235237 }
236238
237239 go func(feedURL, siteURL string) {
238- if f := feed.ResolveFavicon(context.Background(), feedURL, siteURL, ""); f != "" {
239- _ = s.db.UpdateFeedFavicon(context.Background(), feedURL, f)
240+ if fav := feed.ResolveFavicon(context.Background(), feedURL, siteURL); fav != "" {
241+ _ = s.db.UpdateFeedFavicon(context.Background(), feedURL, fav)
240242 }
241243 }(fu.URL, fu.SiteURL)
242244
@@ -72,30 +72,32 @@ func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
72 return72 return
73 }73 }
74 74
75+ var feedTitle string
76+ var faviconURL string
75 if result != nil {77 if result != nil {
76- f := &db.Feed{78+ feedTitle = result.Feed.Title
77- FeedURL: feedURL,79+ faviconURL = result.Feed.FaviconURL
78- Title: nullString(result.Feed.Title),80+ if faviconURL == "" {
79- SiteURL: nullString(result.Feed.SiteURL),81+ go func() {
80- Description: nullString(result.Feed.Description),82+ if f := feed.ResolveFavicon(context.Background(), feedURL, result.Feed.SiteURL); f != "" {
81- FeedType: nullString(result.Feed.Type),83+ _ = s.db.UpdateFeedFavicon(context.Background(), feedURL, f)
82- }84+ }
83- if err := s.db.UpsertFeed(r.Context(), f); err != nil {85+ }()
84- s.logger.Error("failed to upsert feed", "error", err)
85- http.Error(w, err.Error(), http.StatusInternalServerError)
86- return
87 }86 }
88-
89- go func() {
90- if f := feed.ResolveFavicon(context.Background(), feedURL, result.Feed.SiteURL, result.Feed.FaviconURL); f != "" {
91- _ = s.db.UpdateFeedFavicon(context.Background(), feedURL, f)
92- }
93- }()
94 }87 }
95 88
96- var feedTitle string89+ f := &db.Feed{
97- if result != nil {90+ FeedURL: feedURL,
98- feedTitle = result.Feed.Title91+ Title: nullString(feedTitle),
92+ SiteURL: nullString(result.Feed.SiteURL),
93+ Description: nullString(result.Feed.Description),
94+ FeedType: nullString(result.Feed.Type),
95+ FaviconURL: nullString(faviconURL),
96+ }
97+ if err := s.db.UpsertFeed(r.Context(), f); err != nil {
98+ s.logger.Error("failed to upsert feed", "error", err)
99+ http.Error(w, err.Error(), http.StatusInternalServerError)
100+ return
99 }101 }
100 102
101 var subURI, subCID string103 var subURI, subCID string
@@ -235,8 +237,8 @@ func (s *Server) handleOPMLUpload(w http.ResponseWriter, r *http.Request) {
235 }237 }
236 238
237 go func(feedURL, siteURL string) {239 go func(feedURL, siteURL string) {
238- if f := feed.ResolveFavicon(context.Background(), feedURL, siteURL, ""); f != "" {240+ if fav := feed.ResolveFavicon(context.Background(), feedURL, siteURL); fav != "" {
239- _ = s.db.UpdateFeedFavicon(context.Background(), feedURL, f)241+ _ = s.db.UpdateFeedFavicon(context.Background(), feedURL, fav)
240 }242 }
241 }(fu.URL, fu.SiteURL)243 }(fu.URL, fu.SiteURL)
242 244
modified main.go +2 -1
@@ -25,6 +25,7 @@ func main() {
2525 jetstreamURL := flag.String("jetstream", envOr("GLEAN_JETSTREAM", "wss://jetstream.glean.at"), "Jetstream URL")
2626 syncInterval := flag.Duration("sync-interval", envDuration("GLEAN_SYNC_INTERVAL", 1*time.Hour), "PDS sync interval")
2727 clusterInterval := flag.Duration("cluster-interval", envDuration("GLEAN_CLUSTER_INTERVAL", 10*time.Minute), "cluster recomputation interval")
28+ fetchInterval := flag.Duration("fetch-interval", envDuration("GLEAN_FETCH_INTERVAL", 5*time.Minute), "feed fetch tick interval")
2829 collectionDirURL := flag.String("collection-dir", envOr("GLEAN_COLLECTION_DIR_URL", ""), "collection directory URL for startup backfill")
2930 backfillConcurrency := flag.Int("backfill-concurrency", envInt("GLEAN_BACKFILL_CONCURRENCY", 5), "max concurrent backfill workers")
3031 flag.Parse()
@@ -44,7 +45,7 @@ func main() {
4445 callbackURL := envOr("GLEAN_OAUTH_REDIRECT_URL", "")
4546
4647 storeAdapter := db.NewFeedStoreAdapter(database)
47- scheduler := feed.NewScheduler(storeAdapter, logger)
48+ scheduler := feed.NewScheduler(storeAdapter, logger, *fetchInterval, 30*time.Minute)
4849
4950 engine := cluster.NewEngine(database.DB, logger)
5051
@@ -25,6 +25,7 @@ func main() {
25 jetstreamURL := flag.String("jetstream", envOr("GLEAN_JETSTREAM", "wss://jetstream.glean.at"), "Jetstream URL")25 jetstreamURL := flag.String("jetstream", envOr("GLEAN_JETSTREAM", "wss://jetstream.glean.at"), "Jetstream URL")
26 syncInterval := flag.Duration("sync-interval", envDuration("GLEAN_SYNC_INTERVAL", 1*time.Hour), "PDS sync interval")26 syncInterval := flag.Duration("sync-interval", envDuration("GLEAN_SYNC_INTERVAL", 1*time.Hour), "PDS sync interval")
27 clusterInterval := flag.Duration("cluster-interval", envDuration("GLEAN_CLUSTER_INTERVAL", 10*time.Minute), "cluster recomputation interval")27 clusterInterval := flag.Duration("cluster-interval", envDuration("GLEAN_CLUSTER_INTERVAL", 10*time.Minute), "cluster recomputation interval")
28+ fetchInterval := flag.Duration("fetch-interval", envDuration("GLEAN_FETCH_INTERVAL", 5*time.Minute), "feed fetch tick interval")
28 collectionDirURL := flag.String("collection-dir", envOr("GLEAN_COLLECTION_DIR_URL", ""), "collection directory URL for startup backfill")29 collectionDirURL := flag.String("collection-dir", envOr("GLEAN_COLLECTION_DIR_URL", ""), "collection directory URL for startup backfill")
29 backfillConcurrency := flag.Int("backfill-concurrency", envInt("GLEAN_BACKFILL_CONCURRENCY", 5), "max concurrent backfill workers")30 backfillConcurrency := flag.Int("backfill-concurrency", envInt("GLEAN_BACKFILL_CONCURRENCY", 5), "max concurrent backfill workers")
30 flag.Parse()31 flag.Parse()
@@ -44,7 +45,7 @@ func main() {
44 callbackURL := envOr("GLEAN_OAUTH_REDIRECT_URL", "")45 callbackURL := envOr("GLEAN_OAUTH_REDIRECT_URL", "")
45 46
46 storeAdapter := db.NewFeedStoreAdapter(database)47 storeAdapter := db.NewFeedStoreAdapter(database)
47- scheduler := feed.NewScheduler(storeAdapter, logger)48+ scheduler := feed.NewScheduler(storeAdapter, logger, *fetchInterval, 30*time.Minute)
48 49
49 engine := cluster.NewEngine(database.DB, logger)50 engine := cluster.NewEngine(database.DB, logger)
50 51
modified readme.md +1 -0
@@ -44,6 +44,7 @@ Then open `http://localhost:8080`.
4444 | `GLEAN_JETSTREAM` | `wss://jetstream.glean.at` | Jetstream WebSocket URL |
4545 | `GLEAN_SYNC_INTERVAL` | `1h` | PDS sync interval (Go duration: `30m`, `2h30m`, etc.) |
4646 | `GLEAN_CLUSTER_INTERVAL` | `10m` | Cluster recomputation interval (Go duration) |
47+| `GLEAN_FETCH_INTERVAL` | `5m` | Feed fetch scheduler tick interval (Go duration) |
4748 | `GLEAN_COLLECTION_DIR_URL` | _(empty)_ | Collection directory URL for startup backfill |
4849 | `GLEAN_PLC_URL` | `https://didplc.glean.at` | PLC directory URL for DID resolution |
4950 | `GLEAN_OAUTH_CLIENT_ID` | _(empty)_ | OAuth client metadata URL (leave empty for localhost dev) |
@@ -44,6 +44,7 @@ Then open `http://localhost:8080`.
44 | `GLEAN_JETSTREAM` | `wss://jetstream.glean.at` | Jetstream WebSocket URL |44 | `GLEAN_JETSTREAM` | `wss://jetstream.glean.at` | Jetstream WebSocket URL |
45 | `GLEAN_SYNC_INTERVAL` | `1h` | PDS sync interval (Go duration: `30m`, `2h30m`, etc.) |45 | `GLEAN_SYNC_INTERVAL` | `1h` | PDS sync interval (Go duration: `30m`, `2h30m`, etc.) |
46 | `GLEAN_CLUSTER_INTERVAL` | `10m` | Cluster recomputation interval (Go duration) |46 | `GLEAN_CLUSTER_INTERVAL` | `10m` | Cluster recomputation interval (Go duration) |
47+| `GLEAN_FETCH_INTERVAL` | `5m` | Feed fetch scheduler tick interval (Go duration) |
47 | `GLEAN_COLLECTION_DIR_URL` | _(empty)_ | Collection directory URL for startup backfill |48 | `GLEAN_COLLECTION_DIR_URL` | _(empty)_ | Collection directory URL for startup backfill |
48 | `GLEAN_PLC_URL` | `https://didplc.glean.at` | PLC directory URL for DID resolution |49 | `GLEAN_PLC_URL` | `https://didplc.glean.at` | PLC directory URL for DID resolution |
49 | `GLEAN_OAUTH_CLIENT_ID` | _(empty)_ | OAuth client metadata URL (leave empty for localhost dev) |50 | `GLEAN_OAUTH_CLIENT_ID` | _(empty)_ | OAuth client metadata URL (leave empty for localhost dev) |