rickub
nandi/gleanpublic⑂ Fork 0
⑂ 787b9de
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Add Prometheus metrics and monitoring endpointsUnverified

Julien Robert committed 2026-04-20T21:36:16+02:00 Browse files
787b9de parent: 566c75d
modified .gitignore +1 -0
@@ -4,6 +4,7 @@
44 *.dll
55 *.so
66 *.dylib
7+glean
78
89 # Test binary, built with `go test -c`
910 *.test
@@ -4,6 +4,7 @@
4 *.dll4 *.dll
5 *.so5 *.so
6 *.dylib6 *.dylib
7+glean
7 8
8 # Test binary, built with `go test -c`9 # Test binary, built with `go test -c`
9 *.test10 *.test
modified docs/design.md +1 -1
@@ -165,7 +165,7 @@ All custom colors live under the `spot` namespace in `tailwind.config.js`. CSS v
165165 | Dashboard | 2/3 + 1/3 grid | Articles + trending/recommendations sidebar |
166166 | Articles | Full-width list | Keyboard nav (j/k/o/m), mark-all-read |
167167 | Article Detail | `max-w-3xl` centered | Content, like/share/read buttons, annotations |
168-| Feeds | 2/3 + 1/3 grid | Feed list with categories + add/import sidebar |
168+| Feeds | 2/3 + 1/3 grid | Feed list with categories + add/import sidebar, refresh button |
169169 | Trending | Full-width list | Like/annotation counts on each article |
170170 | Discover | Mixed grid | Recommendations + people + browse all |
171171 | Annotations | Full-width list | Filter by article URL, load more |
@@ -165,7 +165,7 @@ All custom colors live under the `spot` namespace in `tailwind.config.js`. CSS v
165 | Dashboard | 2/3 + 1/3 grid | Articles + trending/recommendations sidebar |165 | Dashboard | 2/3 + 1/3 grid | Articles + trending/recommendations sidebar |
166 | Articles | Full-width list | Keyboard nav (j/k/o/m), mark-all-read |166 | Articles | Full-width list | Keyboard nav (j/k/o/m), mark-all-read |
167 | Article Detail | `max-w-3xl` centered | Content, like/share/read buttons, annotations |167 | Article Detail | `max-w-3xl` centered | Content, like/share/read buttons, annotations |
168-| Feeds | 2/3 + 1/3 grid | Feed list with categories + add/import sidebar |168+| Feeds | 2/3 + 1/3 grid | Feed list with categories + add/import sidebar, refresh button |
169 | Trending | Full-width list | Like/annotation counts on each article |169 | Trending | Full-width list | Like/annotation counts on each article |
170 | Discover | Mixed grid | Recommendations + people + browse all |170 | Discover | Mixed grid | Recommendations + people + browse all |
171 | Annotations | Full-width list | Filter by article URL, load more |171 | Annotations | Full-width list | Filter by article URL, load more |
modified docs/specs.md +114 -57
@@ -13,7 +13,7 @@ The core idea: your RSS subscriptions are a strong signal about your interests.
1313 | Layer | Technology |
1414 | ---------------- | ---------------------------------- |
1515 | Backend | Go |
16-| Database | SQLite (via `modernc.org/sqlite`) |
16+| Database | SQLite (via `mattn/go-sqlite3`) |
1717 | Frontend | htmx + TailwindCSS |
1818 | Auth | AT Protocol OAuth / DID resolution |
1919 | AT Protocol role | AppView for `at.glean.*` lexicons |
@@ -239,25 +239,30 @@ Glean is first and foremost an RSS reader. It fetches, parses, and stores articl
239239
240240 ### 4.1 Feed Fetching
241241
242-A background scheduler polls subscribed feeds at regular intervals.
242+A background scheduler polls subscribed feeds on a fixed 5-minute tick. Feeds are fetched at most once per cycle regardless of how many users share them.
243243
244244 ```
245245 ┌─────────────────────────┐
246246 │ Feed Scheduler │
247247 │ (background goroutine) │
248248 └────────┬────────────────┘
249- │ every N minutes
249+ │ every 5 min
250250 ┌────────▼────────────────┐
251251 │ Feed Fetcher │
252252 │ │
253253 │ 1. SELECT feeds where │
254- │ next_fetch <= now │
255- │ 2. Respect ETag/If-None-│
254+ │ subscriber_count > 0 │
255+ │ AND not fetched in │
256+ │ last 30 min │
257+ │ 2. Dedup in-flight: │
258+ │ skip if already │
259+ │ being fetched │
260+ │ 3. Respect ETag/If-None│
256261 │ Match / Last-Modified│
257- │ 3. GET feed URL │
258- │ 4. Parse XML/JSON │
259- │ 5. Upsert articles │
260- │ 6. Update feed metadata│
262+ │ 4. GET feed URL │
263+ │ 5. Parse XML/JSON │
264+ │ 6. Upsert articles │
265+ │ 7. Update feed metadata│
261266 └────────┬────────────────┘
262267 │
263268 ┌──────────────┼──────────────┐
@@ -270,19 +275,21 @@ A background scheduler polls subscribed feeds at regular intervals.
270275
271276 ### 4.2 Fetch Schedule
272277
273-Feeds are not all fetched at the same frequency. The scheduler adapts based on:
278+The scheduler uses a single fixed interval with in-flight deduplication:
274279
275-- **Base interval**: Default 30 minutes
276-- **Feed-level override**: User can set per-feed refresh rate (15min / 30min / 1h / 3h / 6h / 12h / daily)
277-- **Adaptive backoff**: If a feed has not published new articles in the last N fetches, increase the interval. If it starts publishing again, decrease back.
280+- **Tick interval**: The scheduler checks for stale feeds every 5 minutes
281+- **Staleness threshold**: Feeds not fetched in the last 30 minutes are eligible
282+- **Subscriber filter**: Only feeds with `subscriber_count > 0` are fetched
283+- **In-flight dedup**: If a feed is already being fetched (e.g., manual refresh and background scheduler overlap), the second caller waits for the first to complete rather than fetching again
278284 - **HTTP cache**: Honor `ETag` and `Last-Modified` headers to skip parsing when nothing changed (304 Not Modified)
279-- **Error backoff**: On failure, double the interval up to 24h, reset on success
285+- **Error tracking**: `error_count` increments on failure, resets to 0 on success. Feeds with high error counts are surfaced as "dead feeds" to the user.
280286
281287 ```sql
282-ALTER TABLE feeds ADD COLUMN fetch_interval_minutes INTEGER NOT NULL DEFAULT 30;
283-ALTER TABLE feeds ADD COLUMN next_fetch_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP;
284-ALTER TABLE feeds ADD COLUMN consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0;
285-ALTER TABLE feeds ADD COLUMN error_count INTEGER NOT NULL DEFAULT 0;
288+-- Feeds are fetched once regardless of subscriber count
289+SELECT ... FROM feeds
290+WHERE subscriber_count > 0
291+ AND (last_fetched_at IS NULL OR last_fetched_at <= :cutoff)
292+ORDER BY last_fetched_at ASC NULLS FIRST
286293 ```
287294
288295 ### 4.3 Feed Parsing
@@ -440,8 +447,11 @@ CREATE TABLE subscriptions (
440447 id INTEGER PRIMARY KEY AUTOINCREMENT,
441448 user_did TEXT NOT NULL REFERENCES users(did),
442449 feed_url TEXT NOT NULL,
450+ title TEXT,
443451 category TEXT,
444452 added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
453+ uri TEXT,
454+ cid TEXT,
445455 UNIQUE(user_did, feed_url)
446456 );
447457
@@ -462,7 +472,14 @@ CREATE TABLE feeds (
462472 feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),
463473 last_fetched_at DATETIME,
464474 last_error TEXT,
465- subscriber_count INTEGER NOT NULL DEFAULT 0
475+ subscriber_count INTEGER NOT NULL DEFAULT 0,
476+ etag TEXT,
477+ last_modified TEXT,
478+ fetch_interval_minutes INTEGER NOT NULL DEFAULT 30,
479+ next_fetch_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
480+ consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
481+ error_count INTEGER NOT NULL DEFAULT 0,
482+ favicon_url TEXT
466483 );
467484 ```
468485
@@ -475,10 +492,13 @@ CREATE TABLE articles (
475492 id INTEGER PRIMARY KEY AUTOINCREMENT,
476493 feed_url TEXT NOT NULL REFERENCES feeds(feed_url),
477494 guid TEXT NOT NULL,
478- title TEXT,
495+ title TEXT NOT NULL DEFAULT '',
479496 url TEXT,
480497 author TEXT,
498+ summary TEXT,
499+ content TEXT,
481500 published DATETIME,
501+ updated DATETIME,
482502 fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
483503 UNIQUE(feed_url, guid)
484504 );
@@ -487,8 +507,24 @@ CREATE INDEX idx_articles_feed ON articles(feed_url);
487507 CREATE INDEX idx_articles_published ON articles(published DESC);
488508 ```
489509
490-### 6.5 Annotations, Likes
510+### 6.5 Read State
511+
512+```sql
513+CREATE TABLE read_state (
514+ user_did TEXT NOT NULL REFERENCES users(did),
515+ article_id INTEGER NOT NULL REFERENCES articles(id),
516+ is_read BOOLEAN NOT NULL DEFAULT 0,
517+ read_at DATETIME,
518+ is_starred BOOLEAN NOT NULL DEFAULT 0,
519+ starred_at DATETIME,
520+ PRIMARY KEY (user_did, article_id)
521+);
522+
523+CREATE INDEX idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0;
524+CREATE INDEX idx_read_state_starred ON read_state(user_did, is_starred) WHERE is_starred = 1;
525+```
491526
527+### 6.6 Annotations, Likes
492528 Local mirror of AT Protocol lexicon records for fast querying.
493529
494530 ```sql
@@ -502,7 +538,8 @@ CREATE TABLE annotations (
502538 note TEXT,
503539 tags TEXT,
504540 rating INTEGER,
505- created_at DATETIME NOT NULL
541+ created_at DATETIME NOT NULL,
542+ cid TEXT
506543 );
507544
508545 CREATE TABLE likes (
@@ -512,11 +549,12 @@ CREATE TABLE likes (
512549 feed_url TEXT NOT NULL,
513550 article_url TEXT NOT NULL,
514551 created_at DATETIME NOT NULL,
552+ cid TEXT,
515553 UNIQUE(author_did, feed_url, article_url)
516554 );
517555 ```
518556
519-### 6.6 Cluster Precomputation
557+### 6.7 Cluster Precomputation
520558
521559 Stores precomputed similarity data to avoid recalculating on every request.
522560
@@ -657,15 +695,19 @@ The server renders HTML fragments that htmx swaps into the page. No JSON API nee
657695 | `/feeds/opml/download` | GET | Export subscriptions as OPML (offboarding) |
658696 | `/feeds/add` | POST | Add a single feed URL |
659697 | `/feeds/remove` | DELETE | Remove a feed |
698+| `/feeds/refresh` | POST | Refresh all subscribed feeds |
699+| `/feeds/clear` | POST | Clear all subscriptions |
660700 | `/articles` | GET | Read articles (paginated, filterable by feed) |
701+| `/articles/{id}` | GET | Article detail view |
702+| `/articles/{id}/read` | POST | Mark article as read |
703+| `/articles/{id}/unread`| POST | Mark article as unread |
704+| `/articles/{id}/like` | POST | Like an article |
705+| `/articles/mark-all-read` | POST | Mark all articles as read |
661706 | `/trending` | GET | Community feed: articles ranked by likes |
662-| `/discover` | GET | Feed recommendations + similar people |
663-| `/discover/feeds` | GET | Recommended feeds |
664-| `/discover/people` | GET | People with similar reading habits |
707+| `/library` | GET | Liked articles and annotations |
708+| `/library/create` | POST | Create annotation on an article |
709+| `/library/{id}/delete` | POST | Delete an annotation |
665710 | `/profile/{did}` | GET | Public profile: their feeds, likes, annotations |
666-| `/articles/{id}/like` | POST | Like an article (amplify into community feed) |
667-| `/annotations` | GET | View your annotations |
668-| `/annotations/create` | POST | Create annotation on an article |
669711
670712 ### 8.2 htmx Patterns
671713
@@ -681,12 +723,14 @@ glean/
681723 ├── main.go # Entry point, wire everything
682724 ├── go.mod
683725 ├── go.sum
726+├── Dockerfile
727+├── Makefile
684728 ├── internal/
685729 │ ├── atproto/
686730 │ │ ├── auth.go # DID resolution, OAuth flow
687731 │ │ ├── client.go # XRPC client (write to user PDS)
688732 │ │ ├── firehose.go # Subscribe to AT Relay firehose
689-│ │ ├── lexicon.go # Lexicon record types + validation
733+│ │ ├── sync.go # PDS record reconciliation
690734 │ │ └── xrpc.go # XRPC query handlers (AppView endpoints)
691735 │ ├── db/
692736 │ │ ├── db.go # SQLite connection, migrations
@@ -694,49 +738,51 @@ glean/
694738 │ │ ├── feed.go # Feed + subscription queries
695739 │ │ ├── article.go # Article queries
696740 │ │ ├── social.go # Like, annotation queries
697-│ │ └── cluster.go # Similarity + recommendation queries
741+│ │ ├── cluster.go # Similarity + recommendation queries
742+│ │ ├── oauth_store.go # OAuth session storage
743+│ │ └── store.go # FeedStore adapter for scheduler
698744 │ ├── feed/
699745 │ │ ├── parser.go # RSS/Atom/JSON feed parser
700-│ │ ├── fetcher.go # Fetch and parse feeds from URLs
746+│ │ ├── fetcher.go # Scheduler with dedup + Fetcher
747+│ │ ├── discover.go # Feed auto-discovery from URLs
701748 │ │ └── opml.go # OPML import/export
749+│ ├── metrics/
750+│ │ └── metrics.go # Prometheus metrics definitions
702751 │ ├── cluster/
703752 │ │ ├── jaccard.go # Jaccard similarity computation
704753 │ │ ├── recommender.go # Feed + people recommendation logic
705754 │ │ └── cron.go # Background recomputation scheduler
706755 │ ├── server/
707756 │ │ ├── server.go # HTTP server, router setup
757+│ │ ├── auth_handler.go # OAuth login/callback
758+│ │ ├── feeds_handler.go # Feed management handlers
759+│ │ ├── articles_handler.go # Article reading handlers
760+│ │ ├── dashboard_handler.go # Dashboard handler
761+│ │ ├── trending_handler.go # Trending handler
762+│ │ ├── library_handler.go # Library (likes + annotations)
763+│ │ ├── profile_handler.go # Public profile handler
708764 │ │ ├── middleware.go # Auth, logging, CSRF middleware
709-│ │ ├── session.go # Session management (cookie + DID)
710-│ │ └── handlers/
711-│ │ ├── dashboard.go
712-│ │ ├── feeds.go
713-│ │ ├── articles.go
714-│ │ ├── trending.go
715-│ │ ├── discover.go
716-│ │ ├── profile.go
717-│ │ ├── annotations.go
718-│ │ └── auth.go
765+│ │ └── session.go # Session management
766+│ ├── sanitize/
767+│ │ └── sanitize.go # HTML sanitization for article content
719768 │ └── tmpl/
720769 │ ├── base.html # Base template with htmx + Tailwind
721-│ ├── partials/
722-│ │ ├── feed-list.html
723-│ │ ├── article-list.html
724-│ │ ├── like-button.html
725-│ │ ├── annotation-card.html
726-│ │ ├── recommendation-card.html
727-│ │ └── profile-card.html
728-│ ├── dashboard.html
729-│ ├── feeds.html
730-│ ├── articles.html
731-│ ├── trending.html
732-│ ├── discover.html
733-│ ├── profile.html
734-│ └── annotations.html
770+│ ├── index.html # Landing page
771+│ ├── login.html # Login page
772+│ ├── dashboard.html # Dashboard
773+│ ├── feeds.html # Feed management
774+│ ├── articles.html # Article listing
775+│ ├── article_detail.html # Article detail
776+│ ├── trending.html # Trending articles
777+│ ├── library.html # Liked articles + annotations
778+│ ├── profile.html # User profile
779+│ └── partials/ # Reusable template fragments
735780 ├── static/
736781 │ ├── input.css # Tailwind input
737782 │ └── output.css # Tailwind compiled output
738783 ├── docs/
739-│ └── design.md # This document
784+│ ├── specs.md # Technical specification (this document)
785+│ └── design.md # Design system
740786 └── tailwind.config.js
741787 ```
742788
@@ -809,6 +855,17 @@ Browser ──GET /discover/feeds──► Server
809855 - Go's `database/sql` interface makes it easy to swap later if needed
810856 - Matches the project's philosophy of simplicity
811857
858+### 12.3 Prometheus Metrics
859+
860+Glean exposes a `/metrics` endpoint for monitoring. Key metrics:
861+
862+- **`glean_feeds_fetched_total`** — Feed fetch attempts labeled by status (`success`, `error`, `not_modified`)
863+- **`glean_feed_fetch_duration_seconds`** — Histogram of feed fetch latency
864+- **`glean_articles_upserted_total`** — Counter of articles stored from feeds
865+- **`glean_firehose_events_total`** — Firehose events labeled by collection and action
866+- **`glean_http_requests_total`** — HTTP request counts labeled by method, path, and status
867+- **`glean_cluster_runs_total`** / **`glean_cluster_duration_seconds`** — Recommendation engine runs and timing
868+
812869 ### 12.3 Why htmx?
813870
814871 - No JavaScript build pipeline
@@ -13,7 +13,7 @@ The core idea: your RSS subscriptions are a strong signal about your interests.
13 | Layer | Technology |13 | Layer | Technology |
14 | ---------------- | ---------------------------------- |14 | ---------------- | ---------------------------------- |
15 | Backend | Go |15 | Backend | Go |
16-| Database | SQLite (via `modernc.org/sqlite`) |16+| Database | SQLite (via `mattn/go-sqlite3`) |
17 | Frontend | htmx + TailwindCSS |17 | Frontend | htmx + TailwindCSS |
18 | Auth | AT Protocol OAuth / DID resolution |18 | Auth | AT Protocol OAuth / DID resolution |
19 | AT Protocol role | AppView for `at.glean.*` lexicons |19 | AT Protocol role | AppView for `at.glean.*` lexicons |
@@ -239,25 +239,30 @@ Glean is first and foremost an RSS reader. It fetches, parses, and stores articl
239 239
240 ### 4.1 Feed Fetching240 ### 4.1 Feed Fetching
241 241
242-A background scheduler polls subscribed feeds at regular intervals.242+A background scheduler polls subscribed feeds on a fixed 5-minute tick. Feeds are fetched at most once per cycle regardless of how many users share them.
243 243
244 ```244 ```
245 ┌─────────────────────────┐245 ┌─────────────────────────┐
246 │ Feed Scheduler │246 │ Feed Scheduler │
247 │ (background goroutine) │247 │ (background goroutine) │
248 └────────┬────────────────┘248 └────────┬────────────────┘
249- │ every N minutes249+ │ every 5 min
250 ┌────────▼────────────────┐250 ┌────────▼────────────────┐
251 │ Feed Fetcher │251 │ Feed Fetcher │
252 │ │252 │ │
253 │ 1. SELECT feeds where │253 │ 1. SELECT feeds where │
254- │ next_fetch <= now │254+ │ subscriber_count > 0 │
255- │ 2. Respect ETag/If-None-│255+ │ AND not fetched in │
256+ │ last 30 min │
257+ │ 2. Dedup in-flight: │
258+ │ skip if already │
259+ │ being fetched │
260+ │ 3. Respect ETag/If-None│
256 │ Match / Last-Modified│261 │ Match / Last-Modified│
257- │ 3. GET feed URL │262+ │ 4. GET feed URL │
258- │ 4. Parse XML/JSON │263+ │ 5. Parse XML/JSON │
259- │ 5. Upsert articles │264+ │ 6. Upsert articles │
260- │ 6. Update feed metadata│265+ │ 7. Update feed metadata│
261 └────────┬────────────────┘266 └────────┬────────────────┘
262 │267 │
263 ┌──────────────┼──────────────┐268 ┌──────────────┼──────────────┐
@@ -270,19 +275,21 @@ A background scheduler polls subscribed feeds at regular intervals.
270 275
271 ### 4.2 Fetch Schedule276 ### 4.2 Fetch Schedule
272 277
273-Feeds are not all fetched at the same frequency. The scheduler adapts based on:278+The scheduler uses a single fixed interval with in-flight deduplication:
274 279
275-- **Base interval**: Default 30 minutes280+- **Tick interval**: The scheduler checks for stale feeds every 5 minutes
276-- **Feed-level override**: User can set per-feed refresh rate (15min / 30min / 1h / 3h / 6h / 12h / daily)281+- **Staleness threshold**: Feeds not fetched in the last 30 minutes are eligible
277-- **Adaptive backoff**: If a feed has not published new articles in the last N fetches, increase the interval. If it starts publishing again, decrease back.282+- **Subscriber filter**: Only feeds with `subscriber_count > 0` are fetched
283+- **In-flight dedup**: If a feed is already being fetched (e.g., manual refresh and background scheduler overlap), the second caller waits for the first to complete rather than fetching again
278 - **HTTP cache**: Honor `ETag` and `Last-Modified` headers to skip parsing when nothing changed (304 Not Modified)284 - **HTTP cache**: Honor `ETag` and `Last-Modified` headers to skip parsing when nothing changed (304 Not Modified)
279-- **Error backoff**: On failure, double the interval up to 24h, reset on success285+- **Error tracking**: `error_count` increments on failure, resets to 0 on success. Feeds with high error counts are surfaced as "dead feeds" to the user.
280 286
281 ```sql287 ```sql
282-ALTER TABLE feeds ADD COLUMN fetch_interval_minutes INTEGER NOT NULL DEFAULT 30;288+-- Feeds are fetched once regardless of subscriber count
283-ALTER TABLE feeds ADD COLUMN next_fetch_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP;289+SELECT ... FROM feeds
284-ALTER TABLE feeds ADD COLUMN consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0;290+WHERE subscriber_count > 0
285-ALTER TABLE feeds ADD COLUMN error_count INTEGER NOT NULL DEFAULT 0;291+ AND (last_fetched_at IS NULL OR last_fetched_at <= :cutoff)
292+ORDER BY last_fetched_at ASC NULLS FIRST
286 ```293 ```
287 294
288 ### 4.3 Feed Parsing295 ### 4.3 Feed Parsing
@@ -440,8 +447,11 @@ CREATE TABLE subscriptions (
440 id INTEGER PRIMARY KEY AUTOINCREMENT,447 id INTEGER PRIMARY KEY AUTOINCREMENT,
441 user_did TEXT NOT NULL REFERENCES users(did),448 user_did TEXT NOT NULL REFERENCES users(did),
442 feed_url TEXT NOT NULL,449 feed_url TEXT NOT NULL,
450+ title TEXT,
443 category TEXT,451 category TEXT,
444 added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,452 added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
453+ uri TEXT,
454+ cid TEXT,
445 UNIQUE(user_did, feed_url)455 UNIQUE(user_did, feed_url)
446 );456 );
447 457
@@ -462,7 +472,14 @@ CREATE TABLE feeds (
462 feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),472 feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),
463 last_fetched_at DATETIME,473 last_fetched_at DATETIME,
464 last_error TEXT,474 last_error TEXT,
465- subscriber_count INTEGER NOT NULL DEFAULT 0475+ subscriber_count INTEGER NOT NULL DEFAULT 0,
476+ etag TEXT,
477+ last_modified TEXT,
478+ fetch_interval_minutes INTEGER NOT NULL DEFAULT 30,
479+ next_fetch_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
480+ consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
481+ error_count INTEGER NOT NULL DEFAULT 0,
482+ favicon_url TEXT
466 );483 );
467 ```484 ```
468 485
@@ -475,10 +492,13 @@ CREATE TABLE articles (
475 id INTEGER PRIMARY KEY AUTOINCREMENT,492 id INTEGER PRIMARY KEY AUTOINCREMENT,
476 feed_url TEXT NOT NULL REFERENCES feeds(feed_url),493 feed_url TEXT NOT NULL REFERENCES feeds(feed_url),
477 guid TEXT NOT NULL,494 guid TEXT NOT NULL,
478- title TEXT,495+ title TEXT NOT NULL DEFAULT '',
479 url TEXT,496 url TEXT,
480 author TEXT,497 author TEXT,
498+ summary TEXT,
499+ content TEXT,
481 published DATETIME,500 published DATETIME,
501+ updated DATETIME,
482 fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,502 fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
483 UNIQUE(feed_url, guid)503 UNIQUE(feed_url, guid)
484 );504 );
@@ -487,8 +507,24 @@ CREATE INDEX idx_articles_feed ON articles(feed_url);
487 CREATE INDEX idx_articles_published ON articles(published DESC);507 CREATE INDEX idx_articles_published ON articles(published DESC);
488 ```508 ```
489 509
490-### 6.5 Annotations, Likes510+### 6.5 Read State
511+
512+```sql
513+CREATE TABLE read_state (
514+ user_did TEXT NOT NULL REFERENCES users(did),
515+ article_id INTEGER NOT NULL REFERENCES articles(id),
516+ is_read BOOLEAN NOT NULL DEFAULT 0,
517+ read_at DATETIME,
518+ is_starred BOOLEAN NOT NULL DEFAULT 0,
519+ starred_at DATETIME,
520+ PRIMARY KEY (user_did, article_id)
521+);
522+
523+CREATE INDEX idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0;
524+CREATE INDEX idx_read_state_starred ON read_state(user_did, is_starred) WHERE is_starred = 1;
525+```
491 526
527+### 6.6 Annotations, Likes
492 Local mirror of AT Protocol lexicon records for fast querying.528 Local mirror of AT Protocol lexicon records for fast querying.
493 529
494 ```sql530 ```sql
@@ -502,7 +538,8 @@ CREATE TABLE annotations (
502 note TEXT,538 note TEXT,
503 tags TEXT,539 tags TEXT,
504 rating INTEGER,540 rating INTEGER,
505- created_at DATETIME NOT NULL541+ created_at DATETIME NOT NULL,
542+ cid TEXT
506 );543 );
507 544
508 CREATE TABLE likes (545 CREATE TABLE likes (
@@ -512,11 +549,12 @@ CREATE TABLE likes (
512 feed_url TEXT NOT NULL,549 feed_url TEXT NOT NULL,
513 article_url TEXT NOT NULL,550 article_url TEXT NOT NULL,
514 created_at DATETIME NOT NULL,551 created_at DATETIME NOT NULL,
552+ cid TEXT,
515 UNIQUE(author_did, feed_url, article_url)553 UNIQUE(author_did, feed_url, article_url)
516 );554 );
517 ```555 ```
518 556
519-### 6.6 Cluster Precomputation557+### 6.7 Cluster Precomputation
520 558
521 Stores precomputed similarity data to avoid recalculating on every request.559 Stores precomputed similarity data to avoid recalculating on every request.
522 560
@@ -657,15 +695,19 @@ The server renders HTML fragments that htmx swaps into the page. No JSON API nee
657 | `/feeds/opml/download` | GET | Export subscriptions as OPML (offboarding) |695 | `/feeds/opml/download` | GET | Export subscriptions as OPML (offboarding) |
658 | `/feeds/add` | POST | Add a single feed URL |696 | `/feeds/add` | POST | Add a single feed URL |
659 | `/feeds/remove` | DELETE | Remove a feed |697 | `/feeds/remove` | DELETE | Remove a feed |
698+| `/feeds/refresh` | POST | Refresh all subscribed feeds |
699+| `/feeds/clear` | POST | Clear all subscriptions |
660 | `/articles` | GET | Read articles (paginated, filterable by feed) |700 | `/articles` | GET | Read articles (paginated, filterable by feed) |
701+| `/articles/{id}` | GET | Article detail view |
702+| `/articles/{id}/read` | POST | Mark article as read |
703+| `/articles/{id}/unread`| POST | Mark article as unread |
704+| `/articles/{id}/like` | POST | Like an article |
705+| `/articles/mark-all-read` | POST | Mark all articles as read |
661 | `/trending` | GET | Community feed: articles ranked by likes |706 | `/trending` | GET | Community feed: articles ranked by likes |
662-| `/discover` | GET | Feed recommendations + similar people |707+| `/library` | GET | Liked articles and annotations |
663-| `/discover/feeds` | GET | Recommended feeds |708+| `/library/create` | POST | Create annotation on an article |
664-| `/discover/people` | GET | People with similar reading habits |709+| `/library/{id}/delete` | POST | Delete an annotation |
665 | `/profile/{did}` | GET | Public profile: their feeds, likes, annotations |710 | `/profile/{did}` | GET | Public profile: their feeds, likes, annotations |
666-| `/articles/{id}/like` | POST | Like an article (amplify into community feed) |
667-| `/annotations` | GET | View your annotations |
668-| `/annotations/create` | POST | Create annotation on an article |
669 711
670 ### 8.2 htmx Patterns712 ### 8.2 htmx Patterns
671 713
@@ -681,12 +723,14 @@ glean/
681 ├── main.go # Entry point, wire everything723 ├── main.go # Entry point, wire everything
682 ├── go.mod724 ├── go.mod
683 ├── go.sum725 ├── go.sum
726+├── Dockerfile
727+├── Makefile
684 ├── internal/728 ├── internal/
685 │ ├── atproto/729 │ ├── atproto/
686 │ │ ├── auth.go # DID resolution, OAuth flow730 │ │ ├── auth.go # DID resolution, OAuth flow
687 │ │ ├── client.go # XRPC client (write to user PDS)731 │ │ ├── client.go # XRPC client (write to user PDS)
688 │ │ ├── firehose.go # Subscribe to AT Relay firehose732 │ │ ├── firehose.go # Subscribe to AT Relay firehose
689-│ │ ├── lexicon.go # Lexicon record types + validation733+│ │ ├── sync.go # PDS record reconciliation
690 │ │ └── xrpc.go # XRPC query handlers (AppView endpoints)734 │ │ └── xrpc.go # XRPC query handlers (AppView endpoints)
691 │ ├── db/735 │ ├── db/
692 │ │ ├── db.go # SQLite connection, migrations736 │ │ ├── db.go # SQLite connection, migrations
@@ -694,49 +738,51 @@ glean/
694 │ │ ├── feed.go # Feed + subscription queries738 │ │ ├── feed.go # Feed + subscription queries
695 │ │ ├── article.go # Article queries739 │ │ ├── article.go # Article queries
696 │ │ ├── social.go # Like, annotation queries740 │ │ ├── social.go # Like, annotation queries
697-│ │ └── cluster.go # Similarity + recommendation queries741+│ │ ├── cluster.go # Similarity + recommendation queries
742+│ │ ├── oauth_store.go # OAuth session storage
743+│ │ └── store.go # FeedStore adapter for scheduler
698 │ ├── feed/744 │ ├── feed/
699 │ │ ├── parser.go # RSS/Atom/JSON feed parser745 │ │ ├── parser.go # RSS/Atom/JSON feed parser
700-│ │ ├── fetcher.go # Fetch and parse feeds from URLs746+│ │ ├── fetcher.go # Scheduler with dedup + Fetcher
747+│ │ ├── discover.go # Feed auto-discovery from URLs
701 │ │ └── opml.go # OPML import/export748 │ │ └── opml.go # OPML import/export
749+│ ├── metrics/
750+│ │ └── metrics.go # Prometheus metrics definitions
702 │ ├── cluster/751 │ ├── cluster/
703 │ │ ├── jaccard.go # Jaccard similarity computation752 │ │ ├── jaccard.go # Jaccard similarity computation
704 │ │ ├── recommender.go # Feed + people recommendation logic753 │ │ ├── recommender.go # Feed + people recommendation logic
705 │ │ └── cron.go # Background recomputation scheduler754 │ │ └── cron.go # Background recomputation scheduler
706 │ ├── server/755 │ ├── server/
707 │ │ ├── server.go # HTTP server, router setup756 │ │ ├── server.go # HTTP server, router setup
757+│ │ ├── auth_handler.go # OAuth login/callback
758+│ │ ├── feeds_handler.go # Feed management handlers
759+│ │ ├── articles_handler.go # Article reading handlers
760+│ │ ├── dashboard_handler.go # Dashboard handler
761+│ │ ├── trending_handler.go # Trending handler
762+│ │ ├── library_handler.go # Library (likes + annotations)
763+│ │ ├── profile_handler.go # Public profile handler
708 │ │ ├── middleware.go # Auth, logging, CSRF middleware764 │ │ ├── middleware.go # Auth, logging, CSRF middleware
709-│ │ ├── session.go # Session management (cookie + DID)765+│ │ └── session.go # Session management
710-│ │ └── handlers/766+│ ├── sanitize/
711-│ │ ├── dashboard.go767+│ │ └── sanitize.go # HTML sanitization for article content
712-│ │ ├── feeds.go
713-│ │ ├── articles.go
714-│ │ ├── trending.go
715-│ │ ├── discover.go
716-│ │ ├── profile.go
717-│ │ ├── annotations.go
718-│ │ └── auth.go
719 │ └── tmpl/768 │ └── tmpl/
720 │ ├── base.html # Base template with htmx + Tailwind769 │ ├── base.html # Base template with htmx + Tailwind
721-│ ├── partials/770+│ ├── index.html # Landing page
722-│ │ ├── feed-list.html771+│ ├── login.html # Login page
723-│ │ ├── article-list.html772+│ ├── dashboard.html # Dashboard
724-│ │ ├── like-button.html773+│ ├── feeds.html # Feed management
725-│ │ ├── annotation-card.html774+│ ├── articles.html # Article listing
726-│ │ ├── recommendation-card.html775+│ ├── article_detail.html # Article detail
727-│ │ └── profile-card.html776+│ ├── trending.html # Trending articles
728-│ ├── dashboard.html777+│ ├── library.html # Liked articles + annotations
729-│ ├── feeds.html778+│ ├── profile.html # User profile
730-│ ├── articles.html779+│ └── partials/ # Reusable template fragments
731-│ ├── trending.html
732-│ ├── discover.html
733-│ ├── profile.html
734-│ └── annotations.html
735 ├── static/780 ├── static/
736 │ ├── input.css # Tailwind input781 │ ├── input.css # Tailwind input
737 │ └── output.css # Tailwind compiled output782 │ └── output.css # Tailwind compiled output
738 ├── docs/783 ├── docs/
739-│ └── design.md # This document784+│ ├── specs.md # Technical specification (this document)
785+│ └── design.md # Design system
740 └── tailwind.config.js786 └── tailwind.config.js
741 ```787 ```
742 788
@@ -809,6 +855,17 @@ Browser ──GET /discover/feeds──► Server
809 - Go's `database/sql` interface makes it easy to swap later if needed855 - Go's `database/sql` interface makes it easy to swap later if needed
810 - Matches the project's philosophy of simplicity856 - Matches the project's philosophy of simplicity
811 857
858+### 12.3 Prometheus Metrics
859+
860+Glean exposes a `/metrics` endpoint for monitoring. Key metrics:
861+
862+- **`glean_feeds_fetched_total`** — Feed fetch attempts labeled by status (`success`, `error`, `not_modified`)
863+- **`glean_feed_fetch_duration_seconds`** — Histogram of feed fetch latency
864+- **`glean_articles_upserted_total`** — Counter of articles stored from feeds
865+- **`glean_firehose_events_total`** — Firehose events labeled by collection and action
866+- **`glean_http_requests_total`** — HTTP request counts labeled by method, path, and status
867+- **`glean_cluster_runs_total`** / **`glean_cluster_duration_seconds`** — Recommendation engine runs and timing
868+
812 ### 12.3 Why htmx?869 ### 12.3 Why htmx?
813 870
814 - No JavaScript build pipeline871 - No JavaScript build pipeline
modified internal/atproto/firehose.go +6 -0
@@ -10,6 +10,8 @@ import (
1010 "time"
1111
1212 "github.com/gorilla/websocket"
13+
14+ "pkg.rbrt.fr/glean/internal/metrics"
1315 )
1416
1517 type FirehoseEvent struct {
@@ -54,6 +56,7 @@ func (fc *FirehoseConsumer) Start(ctx context.Context) error {
5456 }
5557 if err != nil {
5658 fc.logger.Error("firehose connection error", "error", err)
59+ metrics.FirehoseReconnects.Inc()
5760 }
5861
5962 select {
@@ -181,7 +184,10 @@ func (fc *FirehoseConsumer) parseCommit(ctx context.Context, raw json.RawMessage
181184
182185 if err := fc.handler(ctx, evt); err != nil {
183186 fc.logger.Error("firehose handler error", "error", err)
187+ metrics.FirehoseErrors.Inc()
184188 }
189+
190+ metrics.FirehoseEvents.WithLabelValues(collection, action).Inc()
185191 }
186192 }
187193
@@ -10,6 +10,8 @@ import (
10 "time"10 "time"
11 11
12 "github.com/gorilla/websocket"12 "github.com/gorilla/websocket"
13+
14+ "pkg.rbrt.fr/glean/internal/metrics"
13 )15 )
14 16
15 type FirehoseEvent struct {17 type FirehoseEvent struct {
@@ -54,6 +56,7 @@ func (fc *FirehoseConsumer) Start(ctx context.Context) error {
54 }56 }
55 if err != nil {57 if err != nil {
56 fc.logger.Error("firehose connection error", "error", err)58 fc.logger.Error("firehose connection error", "error", err)
59+ metrics.FirehoseReconnects.Inc()
57 }60 }
58 61
59 select {62 select {
@@ -181,7 +184,10 @@ func (fc *FirehoseConsumer) parseCommit(ctx context.Context, raw json.RawMessage
181 184
182 if err := fc.handler(ctx, evt); err != nil {185 if err := fc.handler(ctx, evt); err != nil {
183 fc.logger.Error("firehose handler error", "error", err)186 fc.logger.Error("firehose handler error", "error", err)
187+ metrics.FirehoseErrors.Inc()
184 }188 }
189+
190+ metrics.FirehoseEvents.WithLabelValues(collection, action).Inc()
185 }191 }
186 }192 }
187 193
modified internal/cluster/cron.go +5 -0
@@ -4,6 +4,8 @@ import (
44 "context"
55 "log/slog"
66 "time"
7+
8+ "pkg.rbrt.fr/glean/internal/metrics"
79 )
810
911 type Cron struct {
@@ -19,6 +21,7 @@ func NewCron(engine *Engine, interval time.Duration, logger *slog.Logger) *Cron
1921 func (c *Cron) Run(ctx context.Context) error {
2022 for {
2123 c.logger.Info("starting similarity computation")
24+ start := time.Now()
2225
2326 if err := c.engine.ComputeFeedSimilarity(ctx); err != nil {
2427 c.logger.Error("feed similarity failed", "error", err)
@@ -32,6 +35,8 @@ func (c *Cron) Run(ctx context.Context) error {
3235 c.logger.Error("recommendations failed", "error", err)
3336 }
3437
38+ metrics.ClusterRuns.Inc()
39+ metrics.ClusterDuration.Observe(time.Since(start).Seconds())
3540 c.logger.Info("similarity computation complete", "next_run", c.interval)
3641
3742 select {
@@ -4,6 +4,8 @@ import (
4 "context"4 "context"
5 "log/slog"5 "log/slog"
6 "time"6 "time"
7+
8+ "pkg.rbrt.fr/glean/internal/metrics"
7 )9 )
8 10
9 type Cron struct {11 type Cron struct {
@@ -19,6 +21,7 @@ func NewCron(engine *Engine, interval time.Duration, logger *slog.Logger) *Cron
19 func (c *Cron) Run(ctx context.Context) error {21 func (c *Cron) Run(ctx context.Context) error {
20 for {22 for {
21 c.logger.Info("starting similarity computation")23 c.logger.Info("starting similarity computation")
24+ start := time.Now()
22 25
23 if err := c.engine.ComputeFeedSimilarity(ctx); err != nil {26 if err := c.engine.ComputeFeedSimilarity(ctx); err != nil {
24 c.logger.Error("feed similarity failed", "error", err)27 c.logger.Error("feed similarity failed", "error", err)
@@ -32,6 +35,8 @@ func (c *Cron) Run(ctx context.Context) error {
32 c.logger.Error("recommendations failed", "error", err)35 c.logger.Error("recommendations failed", "error", err)
33 }36 }
34 37
38+ metrics.ClusterRuns.Inc()
39+ metrics.ClusterDuration.Observe(time.Since(start).Seconds())
35 c.logger.Info("similarity computation complete", "next_run", c.interval)40 c.logger.Info("similarity computation complete", "next_run", c.interval)
36 41
37 select {42 select {
modified internal/feed/fetcher.go +10 -0
@@ -7,6 +7,8 @@ import (
77 "net/http"
88 "sync"
99 "time"
10+
11+ "pkg.rbrt.fr/glean/internal/metrics"
1012 )
1113
1214 type Fetcher struct {
@@ -132,8 +134,11 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) {
132134 close(call.done)
133135 }()
134136
137+ start := time.Now()
135138 result, newEtag, newLastModified, err := s.fetcher.Fetch(ctx, feed.URL, feed.ETag, feed.LastModified)
139+ metrics.FeedsFetchedDuration.Observe(time.Since(start).Seconds())
136140 if err != nil {
141+ metrics.FeedsFetched.WithLabelValues("error").Inc()
137142 s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL)
138143 if updErr := s.store.MarkFeedFetchError(ctx, feed.URL, err.Error()); updErr != nil {
139144 s.logger.Error("failed to update feed fetch error", "error", updErr, "feed", feed.URL)
@@ -142,16 +147,21 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) {
142147 }
143148
144149 if result == nil {
150+ metrics.FeedsFetched.WithLabelValues("not_modified").Inc()
145151 if updErr := s.store.MarkFeedFetched(ctx, feed.URL, feed.ETag, feed.LastModified); updErr != nil {
146152 s.logger.Error("failed to update feed fetch result", "error", updErr, "feed", feed.URL)
147153 }
148154 return
149155 }
150156
157+ metrics.FeedsFetched.WithLabelValues("success").Inc()
158+
151159 for i := range result.Articles {
152160 result.Articles[i].FeedURL = feed.URL
153161 if _, upsertErr := s.store.UpsertArticle(ctx, &result.Articles[i]); upsertErr != nil {
154162 s.logger.Error("failed to upsert article", "error", upsertErr, "url", result.Articles[i].URL)
163+ } else {
164+ metrics.ArticlesUpserted.Inc()
155165 }
156166 }
157167
@@ -7,6 +7,8 @@ import (
7 "net/http"7 "net/http"
8 "sync"8 "sync"
9 "time"9 "time"
10+
11+ "pkg.rbrt.fr/glean/internal/metrics"
10 )12 )
11 13
12 type Fetcher struct {14 type Fetcher struct {
@@ -132,8 +134,11 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) {
132 close(call.done)134 close(call.done)
133 }()135 }()
134 136
137+ start := time.Now()
135 result, newEtag, newLastModified, err := s.fetcher.Fetch(ctx, feed.URL, feed.ETag, feed.LastModified)138 result, newEtag, newLastModified, err := s.fetcher.Fetch(ctx, feed.URL, feed.ETag, feed.LastModified)
139+ metrics.FeedsFetchedDuration.Observe(time.Since(start).Seconds())
136 if err != nil {140 if err != nil {
141+ metrics.FeedsFetched.WithLabelValues("error").Inc()
137 s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL)142 s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL)
138 if updErr := s.store.MarkFeedFetchError(ctx, feed.URL, err.Error()); updErr != nil {143 if updErr := s.store.MarkFeedFetchError(ctx, feed.URL, err.Error()); updErr != nil {
139 s.logger.Error("failed to update feed fetch error", "error", updErr, "feed", feed.URL)144 s.logger.Error("failed to update feed fetch error", "error", updErr, "feed", feed.URL)
@@ -142,16 +147,21 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) {
142 }147 }
143 148
144 if result == nil {149 if result == nil {
150+ metrics.FeedsFetched.WithLabelValues("not_modified").Inc()
145 if updErr := s.store.MarkFeedFetched(ctx, feed.URL, feed.ETag, feed.LastModified); updErr != nil {151 if updErr := s.store.MarkFeedFetched(ctx, feed.URL, feed.ETag, feed.LastModified); updErr != nil {
146 s.logger.Error("failed to update feed fetch result", "error", updErr, "feed", feed.URL)152 s.logger.Error("failed to update feed fetch result", "error", updErr, "feed", feed.URL)
147 }153 }
148 return154 return
149 }155 }
150 156
157+ metrics.FeedsFetched.WithLabelValues("success").Inc()
158+
151 for i := range result.Articles {159 for i := range result.Articles {
152 result.Articles[i].FeedURL = feed.URL160 result.Articles[i].FeedURL = feed.URL
153 if _, upsertErr := s.store.UpsertArticle(ctx, &result.Articles[i]); upsertErr != nil {161 if _, upsertErr := s.store.UpsertArticle(ctx, &result.Articles[i]); upsertErr != nil {
154 s.logger.Error("failed to upsert article", "error", upsertErr, "url", result.Articles[i].URL)162 s.logger.Error("failed to upsert article", "error", upsertErr, "url", result.Articles[i].URL)
163+ } else {
164+ metrics.ArticlesUpserted.Inc()
155 }165 }
156 }166 }
157 167
added internal/metrics/metrics.go +76 -0
new file mode 100644
@@ -0,0 +1,76 @@
1+package metrics
2+
3+import (
4+ "github.com/prometheus/client_golang/prometheus"
5+ "github.com/prometheus/client_golang/prometheus/promauto"
6+)
7+
8+var (
9+ FeedsFetched = promauto.NewCounterVec(prometheus.CounterOpts{
10+ Name: "glean_feeds_fetched_total",
11+ Help: "Total number of feed fetch attempts",
12+ }, []string{"status"})
13+
14+ FeedsFetchedDuration = promauto.NewHistogram(prometheus.HistogramOpts{
15+ Name: "glean_feed_fetch_duration_seconds",
16+ Help: "Time spent fetching a single feed",
17+ Buckets: prometheus.DefBuckets,
18+ })
19+
20+ ArticlesUpserted = promauto.NewCounter(prometheus.CounterOpts{
21+ Name: "glean_articles_upserted_total",
22+ Help: "Total number of articles upserted",
23+ })
24+
25+ FirehoseEvents = promauto.NewCounterVec(prometheus.CounterOpts{
26+ Name: "glean_firehose_events_total",
27+ Help: "Total number of firehose events processed",
28+ }, []string{"collection", "action"})
29+
30+ FirehoseErrors = promauto.NewCounter(prometheus.CounterOpts{
31+ Name: "glean_firehose_errors_total",
32+ Help: "Total number of firehose handler errors",
33+ })
34+
35+ FirehoseReconnects = promauto.NewCounter(prometheus.CounterOpts{
36+ Name: "glean_firehose_reconnects_total",
37+ Help: "Number of firehose reconnections",
38+ })
39+
40+ HTTPRequests = promauto.NewCounterVec(prometheus.CounterOpts{
41+ Name: "glean_http_requests_total",
42+ Help: "Total HTTP requests",
43+ }, []string{"method", "path", "status"})
44+
45+ HTTPRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
46+ Name: "glean_http_request_duration_seconds",
47+ Help: "HTTP request duration",
48+ Buckets: prometheus.DefBuckets,
49+ }, []string{"method", "path"})
50+
51+ ActiveUsers = promauto.NewGauge(prometheus.GaugeOpts{
52+ Name: "glean_users_active_total",
53+ Help: "Number of users with active sessions",
54+ })
55+
56+ ClusterRuns = promauto.NewCounter(prometheus.CounterOpts{
57+ Name: "glean_cluster_runs_total",
58+ Help: "Number of cluster/recommendation computation runs",
59+ })
60+
61+ ClusterDuration = promauto.NewHistogram(prometheus.HistogramOpts{
62+ Name: "glean_cluster_duration_seconds",
63+ Help: "Time spent computing recommendations",
64+ Buckets: []float64{1, 5, 10, 30, 60, 120, 300},
65+ })
66+
67+ SyncRuns = promauto.NewCounter(prometheus.CounterOpts{
68+ Name: "glean_pds_sync_runs_total",
69+ Help: "Number of PDS sync runs",
70+ })
71+
72+ SyncErrors = promauto.NewCounter(prometheus.CounterOpts{
73+ Name: "glean_pds_sync_errors_total",
74+ Help: "Number of PDS sync errors",
75+ })
76+)
new file mode 100644
@@ -0,0 +1,76 @@
1+package metrics
2+
3+import (
4+ "github.com/prometheus/client_golang/prometheus"
5+ "github.com/prometheus/client_golang/prometheus/promauto"
6+)
7+
8+var (
9+ FeedsFetched = promauto.NewCounterVec(prometheus.CounterOpts{
10+ Name: "glean_feeds_fetched_total",
11+ Help: "Total number of feed fetch attempts",
12+ }, []string{"status"})
13+
14+ FeedsFetchedDuration = promauto.NewHistogram(prometheus.HistogramOpts{
15+ Name: "glean_feed_fetch_duration_seconds",
16+ Help: "Time spent fetching a single feed",
17+ Buckets: prometheus.DefBuckets,
18+ })
19+
20+ ArticlesUpserted = promauto.NewCounter(prometheus.CounterOpts{
21+ Name: "glean_articles_upserted_total",
22+ Help: "Total number of articles upserted",
23+ })
24+
25+ FirehoseEvents = promauto.NewCounterVec(prometheus.CounterOpts{
26+ Name: "glean_firehose_events_total",
27+ Help: "Total number of firehose events processed",
28+ }, []string{"collection", "action"})
29+
30+ FirehoseErrors = promauto.NewCounter(prometheus.CounterOpts{
31+ Name: "glean_firehose_errors_total",
32+ Help: "Total number of firehose handler errors",
33+ })
34+
35+ FirehoseReconnects = promauto.NewCounter(prometheus.CounterOpts{
36+ Name: "glean_firehose_reconnects_total",
37+ Help: "Number of firehose reconnections",
38+ })
39+
40+ HTTPRequests = promauto.NewCounterVec(prometheus.CounterOpts{
41+ Name: "glean_http_requests_total",
42+ Help: "Total HTTP requests",
43+ }, []string{"method", "path", "status"})
44+
45+ HTTPRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
46+ Name: "glean_http_request_duration_seconds",
47+ Help: "HTTP request duration",
48+ Buckets: prometheus.DefBuckets,
49+ }, []string{"method", "path"})
50+
51+ ActiveUsers = promauto.NewGauge(prometheus.GaugeOpts{
52+ Name: "glean_users_active_total",
53+ Help: "Number of users with active sessions",
54+ })
55+
56+ ClusterRuns = promauto.NewCounter(prometheus.CounterOpts{
57+ Name: "glean_cluster_runs_total",
58+ Help: "Number of cluster/recommendation computation runs",
59+ })
60+
61+ ClusterDuration = promauto.NewHistogram(prometheus.HistogramOpts{
62+ Name: "glean_cluster_duration_seconds",
63+ Help: "Time spent computing recommendations",
64+ Buckets: []float64{1, 5, 10, 30, 60, 120, 300},
65+ })
66+
67+ SyncRuns = promauto.NewCounter(prometheus.CounterOpts{
68+ Name: "glean_pds_sync_runs_total",
69+ Help: "Number of PDS sync runs",
70+ })
71+
72+ SyncErrors = promauto.NewCounter(prometheus.CounterOpts{
73+ Name: "glean_pds_sync_errors_total",
74+ Help: "Number of PDS sync errors",
75+ })
76+)
modified internal/server/server.go +28 -0
@@ -9,12 +9,14 @@ import (
99 "net/http"
1010 "net/url"
1111 "path/filepath"
12+ "strconv"
1213 "strings"
1314 "time"
1415
1516 "github.com/go-chi/chi/v5"
1617 "github.com/go-chi/chi/v5/middleware"
1718 "github.com/go-chi/cors"
19+ "github.com/prometheus/client_golang/prometheus/promhttp"
1820
1921 oauth "github.com/bluesky-social/indigo/atproto/auth/oauth"
2022 "github.com/bluesky-social/indigo/atproto/syntax"
@@ -22,6 +24,7 @@ import (
2224 "pkg.rbrt.fr/glean/internal/atproto"
2325 "pkg.rbrt.fr/glean/internal/db"
2426 "pkg.rbrt.fr/glean/internal/feed"
27+ "pkg.rbrt.fr/glean/internal/metrics"
2528 "pkg.rbrt.fr/glean/internal/sanitize"
2629 )
2730
@@ -81,6 +84,7 @@ func (s *Server) setupMiddleware() {
8184 s.router.Use(middleware.Logger)
8285 s.router.Use(middleware.Recoverer)
8386 s.router.Use(middleware.Compress(5))
87+ s.router.Use(s.metricsMiddleware)
8488 s.router.Use(cors.Handler(cors.Options{
8589 AllowedOrigins: []string{"*"},
8690 AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
@@ -91,6 +95,27 @@ func (s *Server) setupMiddleware() {
9195 s.router.Use(s.csrfMiddleware)
9296 }
9397
98+func (s *Server) metricsMiddleware(next http.Handler) http.Handler {
99+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
100+ start := time.Now()
101+ ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
102+
103+ next.ServeHTTP(ww, r)
104+
105+ path := normalizeMetricsPath(r.URL.Path)
106+ status := strconv.Itoa(ww.Status())
107+ metrics.HTTPRequests.WithLabelValues(r.Method, path, status).Inc()
108+ metrics.HTTPRequestDuration.WithLabelValues(r.Method, path).Observe(time.Since(start).Seconds())
109+ })
110+}
111+
112+func normalizeMetricsPath(p string) string {
113+ if strings.HasPrefix(p, "/static/") {
114+ return "/static/*"
115+ }
116+ return p
117+}
118+
94119 func (s *Server) setupRoutes() {
95120 s.router.Get("/", s.handleIndex)
96121
@@ -154,6 +179,7 @@ func (s *Server) setupRoutes() {
154179 s.router.Get("/xrpc/at.glean.listFeedLists", xrpc.ListFeedLists)
155180
156181 s.router.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
182+ s.router.Handle("/metrics", promhttp.Handler())
157183 }
158184
159185 func (s *Server) loadTemplates() {
@@ -328,8 +354,10 @@ func (s *Server) runSyncAll(ctx context.Context) {
328354 client := atproto.NewClient(sess.APIClient())
329355 sync := atproto.NewSync(s.db, client, s.logger)
330356 if err := sync.Run(ctx, u.DID); err != nil {
357+ metrics.SyncErrors.Inc()
331358 s.logger.Error("periodic sync failed", "error", err, "did", u.DID)
332359 }
360+ metrics.SyncRuns.Inc()
333361 }
334362 }
335363
@@ -9,12 +9,14 @@ import (
9 "net/http"9 "net/http"
10 "net/url"10 "net/url"
11 "path/filepath"11 "path/filepath"
12+ "strconv"
12 "strings"13 "strings"
13 "time"14 "time"
14 15
15 "github.com/go-chi/chi/v5"16 "github.com/go-chi/chi/v5"
16 "github.com/go-chi/chi/v5/middleware"17 "github.com/go-chi/chi/v5/middleware"
17 "github.com/go-chi/cors"18 "github.com/go-chi/cors"
19+ "github.com/prometheus/client_golang/prometheus/promhttp"
18 20
19 oauth "github.com/bluesky-social/indigo/atproto/auth/oauth"21 oauth "github.com/bluesky-social/indigo/atproto/auth/oauth"
20 "github.com/bluesky-social/indigo/atproto/syntax"22 "github.com/bluesky-social/indigo/atproto/syntax"
@@ -22,6 +24,7 @@ import (
22 "pkg.rbrt.fr/glean/internal/atproto"24 "pkg.rbrt.fr/glean/internal/atproto"
23 "pkg.rbrt.fr/glean/internal/db"25 "pkg.rbrt.fr/glean/internal/db"
24 "pkg.rbrt.fr/glean/internal/feed"26 "pkg.rbrt.fr/glean/internal/feed"
27+ "pkg.rbrt.fr/glean/internal/metrics"
25 "pkg.rbrt.fr/glean/internal/sanitize"28 "pkg.rbrt.fr/glean/internal/sanitize"
26 )29 )
27 30
@@ -81,6 +84,7 @@ func (s *Server) setupMiddleware() {
81 s.router.Use(middleware.Logger)84 s.router.Use(middleware.Logger)
82 s.router.Use(middleware.Recoverer)85 s.router.Use(middleware.Recoverer)
83 s.router.Use(middleware.Compress(5))86 s.router.Use(middleware.Compress(5))
87+ s.router.Use(s.metricsMiddleware)
84 s.router.Use(cors.Handler(cors.Options{88 s.router.Use(cors.Handler(cors.Options{
85 AllowedOrigins: []string{"*"},89 AllowedOrigins: []string{"*"},
86 AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},90 AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
@@ -91,6 +95,27 @@ func (s *Server) setupMiddleware() {
91 s.router.Use(s.csrfMiddleware)95 s.router.Use(s.csrfMiddleware)
92 }96 }
93 97
98+func (s *Server) metricsMiddleware(next http.Handler) http.Handler {
99+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
100+ start := time.Now()
101+ ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
102+
103+ next.ServeHTTP(ww, r)
104+
105+ path := normalizeMetricsPath(r.URL.Path)
106+ status := strconv.Itoa(ww.Status())
107+ metrics.HTTPRequests.WithLabelValues(r.Method, path, status).Inc()
108+ metrics.HTTPRequestDuration.WithLabelValues(r.Method, path).Observe(time.Since(start).Seconds())
109+ })
110+}
111+
112+func normalizeMetricsPath(p string) string {
113+ if strings.HasPrefix(p, "/static/") {
114+ return "/static/*"
115+ }
116+ return p
117+}
118+
94 func (s *Server) setupRoutes() {119 func (s *Server) setupRoutes() {
95 s.router.Get("/", s.handleIndex)120 s.router.Get("/", s.handleIndex)
96 121
@@ -154,6 +179,7 @@ func (s *Server) setupRoutes() {
154 s.router.Get("/xrpc/at.glean.listFeedLists", xrpc.ListFeedLists)179 s.router.Get("/xrpc/at.glean.listFeedLists", xrpc.ListFeedLists)
155 180
156 s.router.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))181 s.router.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
182+ s.router.Handle("/metrics", promhttp.Handler())
157 }183 }
158 184
159 func (s *Server) loadTemplates() {185 func (s *Server) loadTemplates() {
@@ -328,8 +354,10 @@ func (s *Server) runSyncAll(ctx context.Context) {
328 client := atproto.NewClient(sess.APIClient())354 client := atproto.NewClient(sess.APIClient())
329 sync := atproto.NewSync(s.db, client, s.logger)355 sync := atproto.NewSync(s.db, client, s.logger)
330 if err := sync.Run(ctx, u.DID); err != nil {356 if err := sync.Run(ctx, u.DID); err != nil {
357+ metrics.SyncErrors.Inc()
331 s.logger.Error("periodic sync failed", "error", err, "did", u.DID)358 s.logger.Error("periodic sync failed", "error", err, "did", u.DID)
332 }359 }
360+ metrics.SyncRuns.Inc()
333 }361 }
334 }362 }
335 363