nandi/gleanpublic Fork 0
6b9f76a
Commits
Clone
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Linting + Simplify queryUnverified

Julien Robert committed 2026-04-23T23:19:06+02:00 Browse files
6b9f76a parent: 12d6fac
modified internal/atproto/jetstream.go +1 -1
@@ -56,7 +56,7 @@ func (s *jetstreamScheduler) AddWork(ctx context.Context, _ string, evt *models.
5656 RKey: c.RKey,
5757 URI: fmt.Sprintf("at://%s/%s/%s", evt.Did, c.Collection, c.RKey),
5858 CID: c.CID,
59- Value: json.RawMessage(c.Record),
59+ Value: c.Record,
6060 }
6161
6262 if err := s.handler(ctx, e); err != nil {
@@ -56,7 +56,7 @@ func (s *jetstreamScheduler) AddWork(ctx context.Context, _ string, evt *models.
56 RKey: c.RKey,56 RKey: c.RKey,
57 URI: fmt.Sprintf("at://%s/%s/%s", evt.Did, c.Collection, c.RKey),57 URI: fmt.Sprintf("at://%s/%s/%s", evt.Did, c.Collection, c.RKey),
58 CID: c.CID,58 CID: c.CID,
59- Value: json.RawMessage(c.Record),59+ Value: c.Record,
60 }60 }
61 61
62 if err := s.handler(ctx, e); err != nil {62 if err := s.handler(ctx, e); err != nil {
modified internal/atproto/stream_handler.go +18 -12
@@ -11,6 +11,12 @@ import (
1111 "pkg.rbrt.fr/glean/internal/db"
1212 )
1313
14+const (
15+ actionCreate = "create"
16+ actionUpdate = "update"
17+ actionDelete = "delete"
18+)
19+
1420 var sentinelErrors = []error{db.ErrDuplicateSubscription, db.ErrDuplicateLike}
1521
1622 func isSentinel(err error) bool {
@@ -52,7 +58,7 @@ func (h *StreamDBHandler) Handle(ctx context.Context, event *Event) error {
5258
5359 func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) error {
5460 switch event.Type {
55- case "create", "update":
61+ case actionCreate, actionUpdate:
5662 var rec SubscriptionRecord
5763 if err := json.Unmarshal(event.Value, &rec); err != nil {
5864 return err
@@ -68,7 +74,7 @@ func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event)
6874 }
6975 return err
7076
71- case "delete":
77+ case actionDelete:
7278 parsed, ok := ParseRecordURI(event.URI)
7379 if !ok {
7480 return nil
@@ -84,7 +90,7 @@ func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event)
8490
8591 func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error {
8692 switch event.Type {
87- case "create":
93+ case actionCreate:
8894 var rec LikeRecord
8995 if err := json.Unmarshal(event.Value, &rec); err != nil {
9096 return err
@@ -107,7 +113,7 @@ func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error {
107113 }
108114 return err
109115
110- case "delete":
116+ case actionDelete:
111117 return h.articles.DeleteLike(ctx, event.URI)
112118 }
113119 return nil
@@ -115,7 +121,7 @@ func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error {
115121
116122 func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) error {
117123 switch event.Type {
118- case "create":
124+ case actionCreate:
119125 var rec AnnotationRecord
120126 if err := json.Unmarshal(event.Value, &rec); err != nil {
121127 return err
@@ -141,7 +147,7 @@ func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) er
141147 }
142148 return h.articles.CreateAnnotation(ctx, a)
143149
144- case "delete":
150+ case actionDelete:
145151 return h.articles.DeleteAnnotation(ctx, event.URI)
146152 }
147153 return nil
@@ -149,7 +155,7 @@ func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) er
149155
150156 func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error {
151157 switch event.Type {
152- case "create":
158+ case actionCreate:
153159 var rec FollowRecord
154160 if err := json.Unmarshal(event.Value, &rec); err != nil {
155161 return err
@@ -159,7 +165,7 @@ func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error
159165 }
160166 return h.users.UpsertFollow(ctx, event.DID, rec.Subject, event.URI, event.CID)
161167
162- case "delete":
168+ case actionDelete:
163169 return h.users.DeleteFollowByURI(ctx, event.URI)
164170 }
165171 return nil
@@ -167,7 +173,7 @@ func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error
167173
168174 func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) error {
169175 switch event.Type {
170- case "create", "update":
176+ case actionCreate, actionUpdate:
171177 var rec MarginNoteRecord
172178 if err := json.Unmarshal(event.Value, &rec); err != nil {
173179 return err
@@ -194,14 +200,14 @@ func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) er
194200 }
195201 return h.articles.CreateAnnotation(ctx, a)
196202
197- case "delete":
203+ case actionDelete:
198204 }
199205 return nil
200206 }
201207
202208 func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event *Event) error {
203209 switch event.Type {
204- case "create", "update":
210+ case actionCreate, actionUpdate:
205211 var rec SkyreaderSubscriptionRecord
206212 if err := json.Unmarshal(event.Value, &rec); err != nil {
207213 return err
@@ -217,7 +223,7 @@ func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event
217223 }
218224 return err
219225
220- case "delete":
226+ case actionDelete:
221227 }
222228 return nil
223229 }
@@ -11,6 +11,12 @@ import (
11 "pkg.rbrt.fr/glean/internal/db"11 "pkg.rbrt.fr/glean/internal/db"
12 )12 )
13 13
14+const (
15+ actionCreate = "create"
16+ actionUpdate = "update"
17+ actionDelete = "delete"
18+)
19+
14 var sentinelErrors = []error{db.ErrDuplicateSubscription, db.ErrDuplicateLike}20 var sentinelErrors = []error{db.ErrDuplicateSubscription, db.ErrDuplicateLike}
15 21
16 func isSentinel(err error) bool {22 func isSentinel(err error) bool {
@@ -52,7 +58,7 @@ func (h *StreamDBHandler) Handle(ctx context.Context, event *Event) error {
52 58
53 func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) error {59 func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) error {
54 switch event.Type {60 switch event.Type {
55- case "create", "update":61+ case actionCreate, actionUpdate:
56 var rec SubscriptionRecord62 var rec SubscriptionRecord
57 if err := json.Unmarshal(event.Value, &rec); err != nil {63 if err := json.Unmarshal(event.Value, &rec); err != nil {
58 return err64 return err
@@ -68,7 +74,7 @@ func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event)
68 }74 }
69 return err75 return err
70 76
71- case "delete":77+ case actionDelete:
72 parsed, ok := ParseRecordURI(event.URI)78 parsed, ok := ParseRecordURI(event.URI)
73 if !ok {79 if !ok {
74 return nil80 return nil
@@ -84,7 +90,7 @@ func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event)
84 90
85 func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error {91 func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error {
86 switch event.Type {92 switch event.Type {
87- case "create":93+ case actionCreate:
88 var rec LikeRecord94 var rec LikeRecord
89 if err := json.Unmarshal(event.Value, &rec); err != nil {95 if err := json.Unmarshal(event.Value, &rec); err != nil {
90 return err96 return err
@@ -107,7 +113,7 @@ func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error {
107 }113 }
108 return err114 return err
109 115
110- case "delete":116+ case actionDelete:
111 return h.articles.DeleteLike(ctx, event.URI)117 return h.articles.DeleteLike(ctx, event.URI)
112 }118 }
113 return nil119 return nil
@@ -115,7 +121,7 @@ func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error {
115 121
116 func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) error {122 func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) error {
117 switch event.Type {123 switch event.Type {
118- case "create":124+ case actionCreate:
119 var rec AnnotationRecord125 var rec AnnotationRecord
120 if err := json.Unmarshal(event.Value, &rec); err != nil {126 if err := json.Unmarshal(event.Value, &rec); err != nil {
121 return err127 return err
@@ -141,7 +147,7 @@ func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) er
141 }147 }
142 return h.articles.CreateAnnotation(ctx, a)148 return h.articles.CreateAnnotation(ctx, a)
143 149
144- case "delete":150+ case actionDelete:
145 return h.articles.DeleteAnnotation(ctx, event.URI)151 return h.articles.DeleteAnnotation(ctx, event.URI)
146 }152 }
147 return nil153 return nil
@@ -149,7 +155,7 @@ func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) er
149 155
150 func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error {156 func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error {
151 switch event.Type {157 switch event.Type {
152- case "create":158+ case actionCreate:
153 var rec FollowRecord159 var rec FollowRecord
154 if err := json.Unmarshal(event.Value, &rec); err != nil {160 if err := json.Unmarshal(event.Value, &rec); err != nil {
155 return err161 return err
@@ -159,7 +165,7 @@ func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error
159 }165 }
160 return h.users.UpsertFollow(ctx, event.DID, rec.Subject, event.URI, event.CID)166 return h.users.UpsertFollow(ctx, event.DID, rec.Subject, event.URI, event.CID)
161 167
162- case "delete":168+ case actionDelete:
163 return h.users.DeleteFollowByURI(ctx, event.URI)169 return h.users.DeleteFollowByURI(ctx, event.URI)
164 }170 }
165 return nil171 return nil
@@ -167,7 +173,7 @@ func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error
167 173
168 func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) error {174 func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) error {
169 switch event.Type {175 switch event.Type {
170- case "create", "update":176+ case actionCreate, actionUpdate:
171 var rec MarginNoteRecord177 var rec MarginNoteRecord
172 if err := json.Unmarshal(event.Value, &rec); err != nil {178 if err := json.Unmarshal(event.Value, &rec); err != nil {
173 return err179 return err
@@ -194,14 +200,14 @@ func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) er
194 }200 }
195 return h.articles.CreateAnnotation(ctx, a)201 return h.articles.CreateAnnotation(ctx, a)
196 202
197- case "delete":203+ case actionDelete:
198 }204 }
199 return nil205 return nil
200 }206 }
201 207
202 func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event *Event) error {208 func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event *Event) error {
203 switch event.Type {209 switch event.Type {
204- case "create", "update":210+ case actionCreate, actionUpdate:
205 var rec SkyreaderSubscriptionRecord211 var rec SkyreaderSubscriptionRecord
206 if err := json.Unmarshal(event.Value, &rec); err != nil {212 if err := json.Unmarshal(event.Value, &rec); err != nil {
207 return err213 return err
@@ -217,7 +223,7 @@ func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event
217 }223 }
218 return err224 return err
219 225
220- case "delete":226+ case actionDelete:
221 }227 }
222 return nil228 return nil
223 }229 }
modified internal/atproto/xrpc.go +2 -2
@@ -430,7 +430,7 @@ func (h *XRPCHandler) ListFeedLists(w http.ResponseWriter, r *http.Request) {
430430 var did, feedURL, title string
431431 var cat sql.NullString
432432 if err := subRows.Scan(&did, &feedURL, &title, &cat); err != nil {
433- subRows.Close()
433+ _ = subRows.Close()
434434 http.Error(w, err.Error(), http.StatusInternalServerError)
435435 return
436436 }
@@ -440,7 +440,7 @@ func (h *XRPCHandler) ListFeedLists(w http.ResponseWriter, r *http.Request) {
440440 Category: cat.String,
441441 })
442442 }
443- subRows.Close()
443+ _ = subRows.Close()
444444 }
445445
446446 for _, u := range users {
@@ -430,7 +430,7 @@ func (h *XRPCHandler) ListFeedLists(w http.ResponseWriter, r *http.Request) {
430 var did, feedURL, title string430 var did, feedURL, title string
431 var cat sql.NullString431 var cat sql.NullString
432 if err := subRows.Scan(&did, &feedURL, &title, &cat); err != nil {432 if err := subRows.Scan(&did, &feedURL, &title, &cat); err != nil {
433- subRows.Close()433+ _ = subRows.Close()
434 http.Error(w, err.Error(), http.StatusInternalServerError)434 http.Error(w, err.Error(), http.StatusInternalServerError)
435 return435 return
436 }436 }
@@ -440,7 +440,7 @@ func (h *XRPCHandler) ListFeedLists(w http.ResponseWriter, r *http.Request) {
440 Category: cat.String,440 Category: cat.String,
441 })441 })
442 }442 }
443- subRows.Close()443+ _ = subRows.Close()
444 }444 }
445 445
446 for _, u := range users {446 for _, u := range users {
modified internal/db/article.go +6 -5
@@ -10,6 +10,8 @@ import (
1010 "pkg.rbrt.fr/glean/internal/feed"
1111 )
1212
13+const articlesOrderBy = ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`
14+
1315 type ArticleStore struct {
1416 db *DB
1517 }
@@ -142,7 +144,7 @@ func (s *ArticleStore) ListArticles(ctx context.Context, userDID, feedURL string
142144 }
143145
144146 // Future-published articles (e.g., scheduled) sort last
145- query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`
147+ query += articlesOrderBy
146148 args = append(args, limit, offset)
147149
148150 rows, err := s.db.QueryContext(ctx, query, args...)
@@ -206,7 +208,7 @@ func (s *ArticleStore) ListUnreadArticles(ctx context.Context, userDID, feedURL
206208 }
207209
208210 // Future-published articles (e.g., scheduled) sort last
209- query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`
211+ query += articlesOrderBy
210212 args = append(args, limit, offset)
211213
212214 rows, err := s.db.QueryContext(ctx, query, args...)
@@ -270,7 +272,7 @@ func (s *ArticleStore) ListReadArticles(ctx context.Context, userDID, feedURL st
270272 }
271273
272274 // Future-published articles (e.g., scheduled) sort last
273- query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`
275+ query += articlesOrderBy
274276 args = append(args, limit, offset)
275277
276278 rows, err := s.db.QueryContext(ctx, query, args...)
@@ -339,8 +341,7 @@ func (s *ArticleStore) MarkAllSubscribedRead(ctx context.Context, userDID string
339341 func (s *ArticleStore) GetReadState(ctx context.Context, userDID string, articleID int64) (*ReadState, error) {
340342 rs := &ReadState{}
341343 err := s.db.QueryRowContext(ctx, `
342- SELECT user_did, article_id, is_read, read_at
343- FROM articles.read_state WHERE user_did = ? AND article_id = ?
344+ SELECT * FROM articles.read_state WHERE user_did = ? AND article_id = ?
344345 `, userDID, articleID).Scan(&rs.UserDID, &rs.ArticleID, &rs.IsRead, &rs.ReadAt)
345346 if err == sql.ErrNoRows {
346347 return &ReadState{UserDID: userDID, ArticleID: articleID}, nil
@@ -10,6 +10,8 @@ import (
10 "pkg.rbrt.fr/glean/internal/feed"10 "pkg.rbrt.fr/glean/internal/feed"
11 )11 )
12 12
13+const articlesOrderBy = ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`
14+
13 type ArticleStore struct {15 type ArticleStore struct {
14 db *DB16 db *DB
15 }17 }
@@ -142,7 +144,7 @@ func (s *ArticleStore) ListArticles(ctx context.Context, userDID, feedURL string
142 }144 }
143 145
144 // Future-published articles (e.g., scheduled) sort last146 // Future-published articles (e.g., scheduled) sort last
145- query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`147+ query += articlesOrderBy
146 args = append(args, limit, offset)148 args = append(args, limit, offset)
147 149
148 rows, err := s.db.QueryContext(ctx, query, args...)150 rows, err := s.db.QueryContext(ctx, query, args...)
@@ -206,7 +208,7 @@ func (s *ArticleStore) ListUnreadArticles(ctx context.Context, userDID, feedURL
206 }208 }
207 209
208 // Future-published articles (e.g., scheduled) sort last210 // Future-published articles (e.g., scheduled) sort last
209- query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`211+ query += articlesOrderBy
210 args = append(args, limit, offset)212 args = append(args, limit, offset)
211 213
212 rows, err := s.db.QueryContext(ctx, query, args...)214 rows, err := s.db.QueryContext(ctx, query, args...)
@@ -270,7 +272,7 @@ func (s *ArticleStore) ListReadArticles(ctx context.Context, userDID, feedURL st
270 }272 }
271 273
272 // Future-published articles (e.g., scheduled) sort last274 // Future-published articles (e.g., scheduled) sort last
273- query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`275+ query += articlesOrderBy
274 args = append(args, limit, offset)276 args = append(args, limit, offset)
275 277
276 rows, err := s.db.QueryContext(ctx, query, args...)278 rows, err := s.db.QueryContext(ctx, query, args...)
@@ -339,8 +341,7 @@ func (s *ArticleStore) MarkAllSubscribedRead(ctx context.Context, userDID string
339 func (s *ArticleStore) GetReadState(ctx context.Context, userDID string, articleID int64) (*ReadState, error) {341 func (s *ArticleStore) GetReadState(ctx context.Context, userDID string, articleID int64) (*ReadState, error) {
340 rs := &ReadState{}342 rs := &ReadState{}
341 err := s.db.QueryRowContext(ctx, `343 err := s.db.QueryRowContext(ctx, `
342- SELECT user_did, article_id, is_read, read_at344+ SELECT * FROM articles.read_state WHERE user_did = ? AND article_id = ?
343- FROM articles.read_state WHERE user_did = ? AND article_id = ?
344 `, userDID, articleID).Scan(&rs.UserDID, &rs.ArticleID, &rs.IsRead, &rs.ReadAt)345 `, userDID, articleID).Scan(&rs.UserDID, &rs.ArticleID, &rs.IsRead, &rs.ReadAt)
345 if err == sql.ErrNoRows {346 if err == sql.ErrNoRows {
346 return &ReadState{UserDID: userDID, ArticleID: articleID}, nil347 return &ReadState{UserDID: userDID, ArticleID: articleID}, nil
modified internal/db/db.go +319 -11
@@ -2,8 +2,10 @@ package db
22
33 import (
44 "database/sql"
5+ "fmt"
56 "math"
67 "strings"
8+ "sync/atomic"
79 "time"
810
911 "github.com/mattn/go-sqlite3"
@@ -11,28 +13,338 @@ import (
1113
1214 const DSN = "_journal_mode=WAL&_busy_timeout=30000&_synchronous=NORMAL&_cache=shared"
1315
14-func init() {
15- sql.Register("sqlite3_glean", &sqlite3.SQLiteDriver{
16+type DB struct {
17+ *sql.DB
18+}
19+
20+type Databases struct {
21+ Users *UserStore
22+ Articles *ArticleStore
23+
24+ db *DB
25+}
26+
27+var multiDriverSeq int64
28+
29+func OpenAll(basePath string) (*Databases, error) {
30+ articlesPath := basePath + "_articles"
31+ recsPath := basePath + "_recs"
32+
33+ seq := atomic.AddInt64(&multiDriverSeq, 1)
34+ driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq)
35+
36+ sql.Register(driverName, &sqlite3.SQLiteDriver{
1637 ConnectHook: func(conn *sqlite3.SQLiteConn) error {
17- if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil {
38+ if err := conn.RegisterFunc("exp", math.Exp, true); err != nil {
1839 return err
1940 }
20- if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil {
41+ if err := conn.RegisterFunc("log", math.Log, true); err != nil {
2142 return err
2243 }
23- pragmas := []string{
44+ for _, p := range []string{
2445 `PRAGMA wal_autocheckpoint = 1000`,
2546 `PRAGMA temp_store = MEMORY`,
2647 `PRAGMA mmap_size = 268435456`,
27- }
28- for _, p := range pragmas {
48+ } {
2949 if _, err := conn.Exec(p, nil); err != nil {
3050 return err
3151 }
3252 }
53+ if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil {
54+ return err
55+ }
56+ if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil {
57+ return err
58+ }
3359 return nil
3460 },
3561 })
62+
63+ db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN)
64+ if err != nil {
65+ return nil, err
66+ }
67+ db.SetMaxOpenConns(10)
68+ db.SetMaxIdleConns(5)
69+ db.SetConnMaxLifetime(30 * time.Minute)
70+ d := &DB{db}
71+
72+ if err := initUsersSchema(d); err != nil {
73+ d.Close()
74+ return nil, err
75+ }
76+
77+ if err := initArticlesSchema(d); err != nil {
78+ d.Close()
79+ return nil, err
80+ }
81+
82+ if err := initRecsSchema(d); err != nil {
83+ d.Close()
84+ return nil, err
85+ }
86+
87+ return &Databases{
88+ Users: NewUserStore(d),
89+ Articles: NewArticleStore(d),
90+ db: d,
91+ }, nil
92+}
93+
94+func (d *Databases) Close() error {
95+ if d.db != nil {
96+ _ = d.db.Close()
97+ }
98+ return nil
99+}
100+
101+func (d *Databases) DB() *sql.DB {
102+ return d.db.DB
103+}
104+
105+func initUsersSchema(db *DB) error {
106+ for _, s := range usersSchema {
107+ if _, err := db.Exec(s); err != nil {
108+ return err
109+ }
110+ }
111+ return nil
112+}
113+
114+func initArticlesSchema(db *DB) error {
115+ for _, s := range articlesSchema {
116+ if _, err := db.Exec(s); err != nil {
117+ return err
118+ }
119+ }
120+ return nil
121+}
122+
123+func initRecsSchema(db *DB) error {
124+ for _, s := range recsSchema {
125+ if _, err := db.Exec(s); err != nil {
126+ return err
127+ }
128+ }
129+ return nil
130+}
131+
132+var usersSchema = []string{
133+ `CREATE TABLE IF NOT EXISTS users (
134+ did TEXT PRIMARY KEY,
135+ indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
136+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
137+ )`,
138+
139+ `CREATE TABLE IF NOT EXISTS follows (
140+ user_did TEXT NOT NULL,
141+ target_did TEXT NOT NULL,
142+ uri TEXT,
143+ cid TEXT,
144+ followed_at DATETIME,
145+ PRIMARY KEY (user_did, target_did)
146+ )`,
147+
148+ `CREATE TABLE IF NOT EXISTS oauth_auth_requests (
149+ state TEXT PRIMARY KEY,
150+ data TEXT NOT NULL
151+ )`,
152+
153+ `CREATE TABLE IF NOT EXISTS oauth_sessions (
154+ account_did TEXT NOT NULL,
155+ session_id TEXT NOT NULL,
156+ data TEXT NOT NULL,
157+ PRIMARY KEY (account_did, session_id)
158+ )`,
159+
160+ `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`,
161+ `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
162+ `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
163+ `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
164+}
165+
166+var articlesSchema = []string{
167+ `CREATE TABLE IF NOT EXISTS articles.feeds (
168+ feed_url TEXT PRIMARY KEY,
169+ title TEXT,
170+ site_url TEXT,
171+ description TEXT,
172+ feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),
173+ last_fetched_at DATETIME,
174+ last_error TEXT,
175+ subscriber_count INTEGER NOT NULL DEFAULT 0,
176+ etag TEXT,
177+ last_modified TEXT,
178+ consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
179+ error_count INTEGER NOT NULL DEFAULT 0,
180+ favicon_url TEXT
181+ )`,
182+
183+ `CREATE TABLE IF NOT EXISTS articles.subscriptions (
184+ id INTEGER PRIMARY KEY AUTOINCREMENT,
185+ user_did TEXT NOT NULL,
186+ feed_url TEXT NOT NULL,
187+ title TEXT,
188+ category TEXT,
189+ added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
190+ uri TEXT,
191+ cid TEXT,
192+ UNIQUE(user_did, feed_url)
193+ )`,
194+
195+ `CREATE TABLE IF NOT EXISTS articles.articles (
196+ id INTEGER PRIMARY KEY AUTOINCREMENT,
197+ feed_url TEXT NOT NULL,
198+ guid TEXT NOT NULL,
199+ title TEXT NOT NULL DEFAULT '',
200+ url TEXT,
201+ author TEXT,
202+ summary TEXT,
203+ content TEXT,
204+ full_content TEXT,
205+ published DATETIME,
206+ updated DATETIME,
207+ fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
208+ UNIQUE(feed_url, guid)
209+ )`,
210+
211+ `CREATE TABLE IF NOT EXISTS articles.read_state (
212+ user_did TEXT NOT NULL,
213+ article_id INTEGER NOT NULL,
214+ is_read BOOLEAN NOT NULL DEFAULT 0,
215+ read_at DATETIME,
216+ PRIMARY KEY (user_did, article_id)
217+ )`,
218+
219+ `CREATE TABLE IF NOT EXISTS articles.annotations (
220+ id INTEGER PRIMARY KEY AUTOINCREMENT,
221+ uri TEXT NOT NULL UNIQUE,
222+ author_did TEXT NOT NULL,
223+ feed_url TEXT NOT NULL,
224+ article_url TEXT NOT NULL,
225+ quote TEXT,
226+ note TEXT,
227+ tags TEXT,
228+ rating INTEGER,
229+ created_at DATETIME NOT NULL,
230+ cid TEXT
231+ )`,
232+
233+ `CREATE TABLE IF NOT EXISTS articles.likes (
234+ id INTEGER PRIMARY KEY AUTOINCREMENT,
235+ uri TEXT NOT NULL UNIQUE,
236+ author_did TEXT NOT NULL,
237+ feed_url TEXT NOT NULL,
238+ article_url TEXT NOT NULL,
239+ created_at DATETIME NOT NULL,
240+ cid TEXT,
241+ UNIQUE(author_did, feed_url, article_url)
242+ )`,
243+
244+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`,
245+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
246+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`,
247+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`,
248+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
249+ `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`,
250+ `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`,
251+ `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`,
252+ `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`,
253+ `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`,
254+ `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`,
255+ `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`,
256+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`,
257+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`,
258+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`,
259+
260+ `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,
261+ `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN
262+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
263+ END`,
264+ `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN
265+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
266+ END`,
267+ `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN
268+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
269+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
270+ END`,
271+}
272+
273+var recsSchema = []string{
274+ `CREATE TABLE IF NOT EXISTS recs.feed_similarity (
275+ feed_a TEXT NOT NULL,
276+ feed_b TEXT NOT NULL,
277+ jaccard REAL NOT NULL,
278+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
279+ PRIMARY KEY (feed_a, feed_b),
280+ CHECK(feed_a < feed_b)
281+ )`,
282+
283+ `CREATE TABLE IF NOT EXISTS recs.user_similarity (
284+ user_a TEXT NOT NULL,
285+ user_b TEXT NOT NULL,
286+ jaccard REAL NOT NULL,
287+ common_feeds INTEGER NOT NULL,
288+ common_likes INTEGER NOT NULL DEFAULT 0,
289+ common_tags INTEGER NOT NULL DEFAULT 0,
290+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
291+ PRIMARY KEY (user_a, user_b),
292+ CHECK(user_a < user_b)
293+ )`,
294+
295+ `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations (
296+ user_did TEXT NOT NULL,
297+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
298+ target_id TEXT NOT NULL,
299+ reason TEXT,
300+ dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
301+ PRIMARY KEY (user_did, target_type, target_id)
302+ )`,
303+
304+ `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions (
305+ user_did TEXT NOT NULL,
306+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
307+ target_id TEXT NOT NULL,
308+ first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
309+ last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
310+ shown_count INTEGER NOT NULL DEFAULT 1,
311+ acted BOOLEAN NOT NULL DEFAULT 0,
312+ PRIMARY KEY (user_did, target_type, target_id)
313+ )`,
314+
315+ `CREATE TABLE IF NOT EXISTS recs.follow_distances (
316+ user_a TEXT NOT NULL,
317+ user_b TEXT NOT NULL,
318+ distance INTEGER NOT NULL CHECK(distance IN (1, 2)),
319+ PRIMARY KEY (user_a, user_b)
320+ )`,
321+
322+ `CREATE TABLE IF NOT EXISTS recs.user_signal_weights (
323+ user_did TEXT PRIMARY KEY,
324+ w_sub REAL NOT NULL DEFAULT 1.0,
325+ w_like REAL NOT NULL DEFAULT 0.5,
326+ w_tag REAL NOT NULL DEFAULT 0.3,
327+ w_social REAL NOT NULL DEFAULT 0.7,
328+ w_pop REAL NOT NULL DEFAULT 0.2,
329+ w_category REAL NOT NULL DEFAULT 0.4,
330+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
331+ )`,
332+
333+ `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles (
334+ user_did TEXT PRIMARY KEY,
335+ total_likes INTEGER NOT NULL DEFAULT 0,
336+ total_tags INTEGER NOT NULL DEFAULT 0,
337+ top_categories TEXT,
338+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
339+ )`,
340+
341+ `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`,
342+ `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`,
343+ `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
344+ `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`,
345+ `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
346+ `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`,
347+ `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`,
36348 }
37349
38350 func NullStr(s string) sql.NullString {
@@ -53,7 +365,3 @@ func NullStrTags(tags []string) sql.NullString {
53365 }
54366 return sql.NullString{String: strings.Join(tags, ","), Valid: true}
55367 }
56-
57-type DB struct {
58- *sql.DB
59-}
@@ -2,8 +2,10 @@ package db
2 2
3 import (3 import (
4 "database/sql"4 "database/sql"
5+ "fmt"
5 "math"6 "math"
6 "strings"7 "strings"
8+ "sync/atomic"
7 "time"9 "time"
8 10
9 "github.com/mattn/go-sqlite3"11 "github.com/mattn/go-sqlite3"
@@ -11,28 +13,338 @@ import (
11 13
12 const DSN = "_journal_mode=WAL&_busy_timeout=30000&_synchronous=NORMAL&_cache=shared"14 const DSN = "_journal_mode=WAL&_busy_timeout=30000&_synchronous=NORMAL&_cache=shared"
13 15
14-func init() {16+type DB struct {
15- sql.Register("sqlite3_glean", &sqlite3.SQLiteDriver{17+ *sql.DB
18+}
19+
20+type Databases struct {
21+ Users *UserStore
22+ Articles *ArticleStore
23+
24+ db *DB
25+}
26+
27+var multiDriverSeq int64
28+
29+func OpenAll(basePath string) (*Databases, error) {
30+ articlesPath := basePath + "_articles"
31+ recsPath := basePath + "_recs"
32+
33+ seq := atomic.AddInt64(&multiDriverSeq, 1)
34+ driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq)
35+
36+ sql.Register(driverName, &sqlite3.SQLiteDriver{
16 ConnectHook: func(conn *sqlite3.SQLiteConn) error {37 ConnectHook: func(conn *sqlite3.SQLiteConn) error {
17- if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil {38+ if err := conn.RegisterFunc("exp", math.Exp, true); err != nil {
18 return err39 return err
19 }40 }
20- if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil {41+ if err := conn.RegisterFunc("log", math.Log, true); err != nil {
21 return err42 return err
22 }43 }
23- pragmas := []string{44+ for _, p := range []string{
24 `PRAGMA wal_autocheckpoint = 1000`,45 `PRAGMA wal_autocheckpoint = 1000`,
25 `PRAGMA temp_store = MEMORY`,46 `PRAGMA temp_store = MEMORY`,
26 `PRAGMA mmap_size = 268435456`,47 `PRAGMA mmap_size = 268435456`,
27- }48+ } {
28- for _, p := range pragmas {
29 if _, err := conn.Exec(p, nil); err != nil {49 if _, err := conn.Exec(p, nil); err != nil {
30 return err50 return err
31 }51 }
32 }52 }
53+ if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil {
54+ return err
55+ }
56+ if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil {
57+ return err
58+ }
33 return nil59 return nil
34 },60 },
35 })61 })
62+
63+ db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN)
64+ if err != nil {
65+ return nil, err
66+ }
67+ db.SetMaxOpenConns(10)
68+ db.SetMaxIdleConns(5)
69+ db.SetConnMaxLifetime(30 * time.Minute)
70+ d := &DB{db}
71+
72+ if err := initUsersSchema(d); err != nil {
73+ d.Close()
74+ return nil, err
75+ }
76+
77+ if err := initArticlesSchema(d); err != nil {
78+ d.Close()
79+ return nil, err
80+ }
81+
82+ if err := initRecsSchema(d); err != nil {
83+ d.Close()
84+ return nil, err
85+ }
86+
87+ return &Databases{
88+ Users: NewUserStore(d),
89+ Articles: NewArticleStore(d),
90+ db: d,
91+ }, nil
92+}
93+
94+func (d *Databases) Close() error {
95+ if d.db != nil {
96+ _ = d.db.Close()
97+ }
98+ return nil
99+}
100+
101+func (d *Databases) DB() *sql.DB {
102+ return d.db.DB
103+}
104+
105+func initUsersSchema(db *DB) error {
106+ for _, s := range usersSchema {
107+ if _, err := db.Exec(s); err != nil {
108+ return err
109+ }
110+ }
111+ return nil
112+}
113+
114+func initArticlesSchema(db *DB) error {
115+ for _, s := range articlesSchema {
116+ if _, err := db.Exec(s); err != nil {
117+ return err
118+ }
119+ }
120+ return nil
121+}
122+
123+func initRecsSchema(db *DB) error {
124+ for _, s := range recsSchema {
125+ if _, err := db.Exec(s); err != nil {
126+ return err
127+ }
128+ }
129+ return nil
130+}
131+
132+var usersSchema = []string{
133+ `CREATE TABLE IF NOT EXISTS users (
134+ did TEXT PRIMARY KEY,
135+ indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
136+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
137+ )`,
138+
139+ `CREATE TABLE IF NOT EXISTS follows (
140+ user_did TEXT NOT NULL,
141+ target_did TEXT NOT NULL,
142+ uri TEXT,
143+ cid TEXT,
144+ followed_at DATETIME,
145+ PRIMARY KEY (user_did, target_did)
146+ )`,
147+
148+ `CREATE TABLE IF NOT EXISTS oauth_auth_requests (
149+ state TEXT PRIMARY KEY,
150+ data TEXT NOT NULL
151+ )`,
152+
153+ `CREATE TABLE IF NOT EXISTS oauth_sessions (
154+ account_did TEXT NOT NULL,
155+ session_id TEXT NOT NULL,
156+ data TEXT NOT NULL,
157+ PRIMARY KEY (account_did, session_id)
158+ )`,
159+
160+ `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`,
161+ `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
162+ `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
163+ `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
164+}
165+
166+var articlesSchema = []string{
167+ `CREATE TABLE IF NOT EXISTS articles.feeds (
168+ feed_url TEXT PRIMARY KEY,
169+ title TEXT,
170+ site_url TEXT,
171+ description TEXT,
172+ feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),
173+ last_fetched_at DATETIME,
174+ last_error TEXT,
175+ subscriber_count INTEGER NOT NULL DEFAULT 0,
176+ etag TEXT,
177+ last_modified TEXT,
178+ consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
179+ error_count INTEGER NOT NULL DEFAULT 0,
180+ favicon_url TEXT
181+ )`,
182+
183+ `CREATE TABLE IF NOT EXISTS articles.subscriptions (
184+ id INTEGER PRIMARY KEY AUTOINCREMENT,
185+ user_did TEXT NOT NULL,
186+ feed_url TEXT NOT NULL,
187+ title TEXT,
188+ category TEXT,
189+ added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
190+ uri TEXT,
191+ cid TEXT,
192+ UNIQUE(user_did, feed_url)
193+ )`,
194+
195+ `CREATE TABLE IF NOT EXISTS articles.articles (
196+ id INTEGER PRIMARY KEY AUTOINCREMENT,
197+ feed_url TEXT NOT NULL,
198+ guid TEXT NOT NULL,
199+ title TEXT NOT NULL DEFAULT '',
200+ url TEXT,
201+ author TEXT,
202+ summary TEXT,
203+ content TEXT,
204+ full_content TEXT,
205+ published DATETIME,
206+ updated DATETIME,
207+ fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
208+ UNIQUE(feed_url, guid)
209+ )`,
210+
211+ `CREATE TABLE IF NOT EXISTS articles.read_state (
212+ user_did TEXT NOT NULL,
213+ article_id INTEGER NOT NULL,
214+ is_read BOOLEAN NOT NULL DEFAULT 0,
215+ read_at DATETIME,
216+ PRIMARY KEY (user_did, article_id)
217+ )`,
218+
219+ `CREATE TABLE IF NOT EXISTS articles.annotations (
220+ id INTEGER PRIMARY KEY AUTOINCREMENT,
221+ uri TEXT NOT NULL UNIQUE,
222+ author_did TEXT NOT NULL,
223+ feed_url TEXT NOT NULL,
224+ article_url TEXT NOT NULL,
225+ quote TEXT,
226+ note TEXT,
227+ tags TEXT,
228+ rating INTEGER,
229+ created_at DATETIME NOT NULL,
230+ cid TEXT
231+ )`,
232+
233+ `CREATE TABLE IF NOT EXISTS articles.likes (
234+ id INTEGER PRIMARY KEY AUTOINCREMENT,
235+ uri TEXT NOT NULL UNIQUE,
236+ author_did TEXT NOT NULL,
237+ feed_url TEXT NOT NULL,
238+ article_url TEXT NOT NULL,
239+ created_at DATETIME NOT NULL,
240+ cid TEXT,
241+ UNIQUE(author_did, feed_url, article_url)
242+ )`,
243+
244+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`,
245+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
246+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`,
247+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`,
248+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
249+ `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`,
250+ `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`,
251+ `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`,
252+ `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`,
253+ `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`,
254+ `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`,
255+ `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`,
256+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`,
257+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`,
258+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`,
259+
260+ `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,
261+ `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN
262+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
263+ END`,
264+ `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN
265+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
266+ END`,
267+ `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN
268+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
269+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
270+ END`,
271+}
272+
273+var recsSchema = []string{
274+ `CREATE TABLE IF NOT EXISTS recs.feed_similarity (
275+ feed_a TEXT NOT NULL,
276+ feed_b TEXT NOT NULL,
277+ jaccard REAL NOT NULL,
278+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
279+ PRIMARY KEY (feed_a, feed_b),
280+ CHECK(feed_a < feed_b)
281+ )`,
282+
283+ `CREATE TABLE IF NOT EXISTS recs.user_similarity (
284+ user_a TEXT NOT NULL,
285+ user_b TEXT NOT NULL,
286+ jaccard REAL NOT NULL,
287+ common_feeds INTEGER NOT NULL,
288+ common_likes INTEGER NOT NULL DEFAULT 0,
289+ common_tags INTEGER NOT NULL DEFAULT 0,
290+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
291+ PRIMARY KEY (user_a, user_b),
292+ CHECK(user_a < user_b)
293+ )`,
294+
295+ `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations (
296+ user_did TEXT NOT NULL,
297+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
298+ target_id TEXT NOT NULL,
299+ reason TEXT,
300+ dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
301+ PRIMARY KEY (user_did, target_type, target_id)
302+ )`,
303+
304+ `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions (
305+ user_did TEXT NOT NULL,
306+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
307+ target_id TEXT NOT NULL,
308+ first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
309+ last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
310+ shown_count INTEGER NOT NULL DEFAULT 1,
311+ acted BOOLEAN NOT NULL DEFAULT 0,
312+ PRIMARY KEY (user_did, target_type, target_id)
313+ )`,
314+
315+ `CREATE TABLE IF NOT EXISTS recs.follow_distances (
316+ user_a TEXT NOT NULL,
317+ user_b TEXT NOT NULL,
318+ distance INTEGER NOT NULL CHECK(distance IN (1, 2)),
319+ PRIMARY KEY (user_a, user_b)
320+ )`,
321+
322+ `CREATE TABLE IF NOT EXISTS recs.user_signal_weights (
323+ user_did TEXT PRIMARY KEY,
324+ w_sub REAL NOT NULL DEFAULT 1.0,
325+ w_like REAL NOT NULL DEFAULT 0.5,
326+ w_tag REAL NOT NULL DEFAULT 0.3,
327+ w_social REAL NOT NULL DEFAULT 0.7,
328+ w_pop REAL NOT NULL DEFAULT 0.2,
329+ w_category REAL NOT NULL DEFAULT 0.4,
330+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
331+ )`,
332+
333+ `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles (
334+ user_did TEXT PRIMARY KEY,
335+ total_likes INTEGER NOT NULL DEFAULT 0,
336+ total_tags INTEGER NOT NULL DEFAULT 0,
337+ top_categories TEXT,
338+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
339+ )`,
340+
341+ `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`,
342+ `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`,
343+ `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
344+ `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`,
345+ `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
346+ `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`,
347+ `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`,
36 }348 }
37 349
38 func NullStr(s string) sql.NullString {350 func NullStr(s string) sql.NullString {
@@ -53,7 +365,3 @@ func NullStrTags(tags []string) sql.NullString {
53 }365 }
54 return sql.NullString{String: strings.Join(tags, ","), Valid: true}366 return sql.NullString{String: strings.Join(tags, ","), Valid: true}
55 }367 }
56-
57-type DB struct {
58- *sql.DB
59-}
modified internal/db/feed.go +5 -8
@@ -54,9 +54,6 @@ type Subscription struct {
5454 FaviconURL sql.NullString
5555 }
5656
57-const feedSelectCols = `feed_url, title, site_url, description, feed_type,
58- last_fetched_at, last_error, subscriber_count, etag, last_modified,
59- consecutive_empty_fetches, error_count, favicon_url`
6057
6158 func scanFeed(scanner interface{ Scan(...any) error }) (*Feed, error) {
6259 f := &Feed{}
@@ -73,13 +70,13 @@ func (s *ArticleStore) UpsertFeed(ctx context.Context, feed *Feed) error {
7370 }
7471
7572 func (s *ArticleStore) GetFeed(ctx context.Context, feedURL string) (*Feed, error) {
76- row := s.db.QueryRowContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds WHERE feed_url = ?`, feedURL)
73+ row := s.db.QueryRowContext(ctx, `SELECT * FROM articles.feeds WHERE feed_url = ?`, feedURL)
7774 return scanFeed(row)
7875 }
7976
8077 func (s *ArticleStore) GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error) {
8178 cutoff := time.Now().Add(-olderThan)
82- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds
79+ rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds
8380 WHERE subscriber_count > 0 AND error_count < 25 AND (last_fetched_at IS NULL OR last_fetched_at <= ?)
8481 ORDER BY last_fetched_at ASC NULLS FIRST LIMIT ?`, cutoff, limit)
8582 if err != nil {
@@ -309,7 +306,7 @@ func (s *ArticleStore) UpdateFeedFavicon(ctx context.Context, feedURL, faviconUR
309306 }
310307
311308 func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, threshold int) ([]*Feed, error) {
312- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds f
309+ rows, err := s.db.QueryContext(ctx, `SELECT f.* FROM articles.feeds f
313310 JOIN articles.subscriptions s ON s.feed_url = f.feed_url AND s.user_did = ?
314311 WHERE f.error_count >= ? ORDER BY f.error_count DESC`, userDID, threshold)
315312 if err != nil {
@@ -329,7 +326,7 @@ func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, thresh
329326 }
330327
331328 func (s *ArticleStore) ListAllFeeds(ctx context.Context, limit, offset int) ([]*Feed, error) {
332- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds
329+ rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds
333330 ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, limit, offset)
334331 if err != nil {
335332 return nil, err
@@ -459,7 +456,7 @@ func (s *ArticleStore) BatchReconcileSubscriptions(ctx context.Context, userDID
459456 }
460457
461458 func (s *ArticleStore) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) {
462- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds
459+ rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds
463460 WHERE feed_url NOT IN (SELECT feed_url FROM articles.subscriptions WHERE user_did = ?)
464461 ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, userDID, limit, offset)
465462 if err != nil {
@@ -54,9 +54,6 @@ type Subscription struct {
54 FaviconURL sql.NullString54 FaviconURL sql.NullString
55 }55 }
56 56
57-const feedSelectCols = `feed_url, title, site_url, description, feed_type,
58- last_fetched_at, last_error, subscriber_count, etag, last_modified,
59- consecutive_empty_fetches, error_count, favicon_url`
60 57
61 func scanFeed(scanner interface{ Scan(...any) error }) (*Feed, error) {58 func scanFeed(scanner interface{ Scan(...any) error }) (*Feed, error) {
62 f := &Feed{}59 f := &Feed{}
@@ -73,13 +70,13 @@ func (s *ArticleStore) UpsertFeed(ctx context.Context, feed *Feed) error {
73 }70 }
74 71
75 func (s *ArticleStore) GetFeed(ctx context.Context, feedURL string) (*Feed, error) {72 func (s *ArticleStore) GetFeed(ctx context.Context, feedURL string) (*Feed, error) {
76- row := s.db.QueryRowContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds WHERE feed_url = ?`, feedURL)73+ row := s.db.QueryRowContext(ctx, `SELECT * FROM articles.feeds WHERE feed_url = ?`, feedURL)
77 return scanFeed(row)74 return scanFeed(row)
78 }75 }
79 76
80 func (s *ArticleStore) GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error) {77 func (s *ArticleStore) GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error) {
81 cutoff := time.Now().Add(-olderThan)78 cutoff := time.Now().Add(-olderThan)
82- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds79+ rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds
83 WHERE subscriber_count > 0 AND error_count < 25 AND (last_fetched_at IS NULL OR last_fetched_at <= ?)80 WHERE subscriber_count > 0 AND error_count < 25 AND (last_fetched_at IS NULL OR last_fetched_at <= ?)
84 ORDER BY last_fetched_at ASC NULLS FIRST LIMIT ?`, cutoff, limit)81 ORDER BY last_fetched_at ASC NULLS FIRST LIMIT ?`, cutoff, limit)
85 if err != nil {82 if err != nil {
@@ -309,7 +306,7 @@ func (s *ArticleStore) UpdateFeedFavicon(ctx context.Context, feedURL, faviconUR
309 }306 }
310 307
311 func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, threshold int) ([]*Feed, error) {308 func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, threshold int) ([]*Feed, error) {
312- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds f309+ rows, err := s.db.QueryContext(ctx, `SELECT f.* FROM articles.feeds f
313 JOIN articles.subscriptions s ON s.feed_url = f.feed_url AND s.user_did = ?310 JOIN articles.subscriptions s ON s.feed_url = f.feed_url AND s.user_did = ?
314 WHERE f.error_count >= ? ORDER BY f.error_count DESC`, userDID, threshold)311 WHERE f.error_count >= ? ORDER BY f.error_count DESC`, userDID, threshold)
315 if err != nil {312 if err != nil {
@@ -329,7 +326,7 @@ func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, thresh
329 }326 }
330 327
331 func (s *ArticleStore) ListAllFeeds(ctx context.Context, limit, offset int) ([]*Feed, error) {328 func (s *ArticleStore) ListAllFeeds(ctx context.Context, limit, offset int) ([]*Feed, error) {
332- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds329+ rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds
333 ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, limit, offset)330 ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, limit, offset)
334 if err != nil {331 if err != nil {
335 return nil, err332 return nil, err
@@ -459,7 +456,7 @@ func (s *ArticleStore) BatchReconcileSubscriptions(ctx context.Context, userDID
459 }456 }
460 457
461 func (s *ArticleStore) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) {458 func (s *ArticleStore) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) {
462- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds459+ rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds
463 WHERE feed_url NOT IN (SELECT feed_url FROM articles.subscriptions WHERE user_did = ?)460 WHERE feed_url NOT IN (SELECT feed_url FROM articles.subscriptions WHERE user_did = ?)
464 ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, userDID, limit, offset)461 ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, userDID, limit, offset)
465 if err != nil {462 if err != nil {
modified internal/db/follow.go +1 -2
@@ -60,8 +60,7 @@ func (s *UserStore) ListFollows(ctx context.Context, userDID string, limit, offs
6060
6161 func (s *UserStore) ListFollowers(ctx context.Context, targetDID string, limit, offset int) ([]*Follow, error) {
6262 rows, err := s.db.QueryContext(ctx, `
63- SELECT user_did, target_did, uri, cid, followed_at
64- FROM follows WHERE target_did = ?
63+ SELECT * FROM follows WHERE target_did = ?
6564 ORDER BY followed_at DESC
6665 LIMIT ? OFFSET ?
6766 `, targetDID, limit, offset)
@@ -60,8 +60,7 @@ func (s *UserStore) ListFollows(ctx context.Context, userDID string, limit, offs
60 60
61 func (s *UserStore) ListFollowers(ctx context.Context, targetDID string, limit, offset int) ([]*Follow, error) {61 func (s *UserStore) ListFollowers(ctx context.Context, targetDID string, limit, offset int) ([]*Follow, error) {
62 rows, err := s.db.QueryContext(ctx, `62 rows, err := s.db.QueryContext(ctx, `
63- SELECT user_did, target_did, uri, cid, followed_at63+ SELECT * FROM follows WHERE target_did = ?
64- FROM follows WHERE target_did = ?
65 ORDER BY followed_at DESC64 ORDER BY followed_at DESC
66 LIMIT ? OFFSET ?65 LIMIT ? OFFSET ?
67 `, targetDID, limit, offset)66 `, targetDID, limit, offset)
deleted internal/db/multi.go +0 -341
deleted file mode 100644
@@ -1,341 +0,0 @@
1-package db
2-
3-import (
4- "database/sql"
5- "fmt"
6- "math"
7- "sync/atomic"
8- "time"
9-
10- "github.com/mattn/go-sqlite3"
11-)
12-
13-type Databases struct {
14- Users *UserStore
15- Articles *ArticleStore
16-
17- db *DB
18-}
19-
20-var multiDriverSeq int64
21-
22-func OpenAll(basePath string) (*Databases, error) {
23- articlesPath := basePath + "_articles"
24- recsPath := basePath + "_recs"
25-
26- seq := atomic.AddInt64(&multiDriverSeq, 1)
27- driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq)
28-
29- sql.Register(driverName, &sqlite3.SQLiteDriver{
30- ConnectHook: func(conn *sqlite3.SQLiteConn) error {
31- if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil {
32- return err
33- }
34- if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil {
35- return err
36- }
37- for _, p := range []string{
38- `PRAGMA wal_autocheckpoint = 1000`,
39- `PRAGMA temp_store = MEMORY`,
40- `PRAGMA mmap_size = 268435456`,
41- } {
42- if _, err := conn.Exec(p, nil); err != nil {
43- return err
44- }
45- }
46- if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil {
47- return err
48- }
49- if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil {
50- return err
51- }
52- return nil
53- },
54- })
55-
56- db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN)
57- if err != nil {
58- return nil, err
59- }
60- db.SetMaxOpenConns(10)
61- db.SetMaxIdleConns(5)
62- db.SetConnMaxLifetime(30 * time.Minute)
63- d := &DB{db}
64-
65- if err := initUsersSchema(d); err != nil {
66- d.Close()
67- return nil, err
68- }
69-
70- if err := initArticlesSchema(d); err != nil {
71- d.Close()
72- return nil, err
73- }
74-
75- if err := initRecsSchema(d); err != nil {
76- d.Close()
77- return nil, err
78- }
79-
80- return &Databases{
81- Users: NewUserStore(d),
82- Articles: NewArticleStore(d),
83- db: d,
84- }, nil
85-}
86-
87-func (d *Databases) Close() error {
88- if d.db != nil {
89- _ = d.db.Close()
90- }
91- return nil
92-}
93-
94-func (d *Databases) DB() *sql.DB {
95- return d.db.DB
96-}
97-
98-func initUsersSchema(db *DB) error {
99- for _, s := range usersSchema {
100- if _, err := db.Exec(s); err != nil {
101- return err
102- }
103- }
104- return nil
105-}
106-
107-func initArticlesSchema(db *DB) error {
108- for _, s := range articlesSchema {
109- if _, err := db.Exec(s); err != nil {
110- return err
111- }
112- }
113- return nil
114-}
115-
116-func initRecsSchema(db *DB) error {
117- for _, s := range recsSchema {
118- if _, err := db.Exec(s); err != nil {
119- return err
120- }
121- }
122- return nil
123-}
124-
125-var usersSchema = []string{
126- `CREATE TABLE IF NOT EXISTS users (
127- did TEXT PRIMARY KEY,
128- indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
129- updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
130- )`,
131-
132- `CREATE TABLE IF NOT EXISTS follows (
133- user_did TEXT NOT NULL,
134- target_did TEXT NOT NULL,
135- uri TEXT,
136- cid TEXT,
137- followed_at DATETIME,
138- PRIMARY KEY (user_did, target_did)
139- )`,
140-
141- `CREATE TABLE IF NOT EXISTS oauth_auth_requests (
142- state TEXT PRIMARY KEY,
143- data TEXT NOT NULL
144- )`,
145-
146- `CREATE TABLE IF NOT EXISTS oauth_sessions (
147- account_did TEXT NOT NULL,
148- session_id TEXT NOT NULL,
149- data TEXT NOT NULL,
150- PRIMARY KEY (account_did, session_id)
151- )`,
152-
153- `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`,
154- `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
155- `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
156- `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
157-}
158-
159-var articlesSchema = []string{
160- `CREATE TABLE IF NOT EXISTS articles.feeds (
161- feed_url TEXT PRIMARY KEY,
162- title TEXT,
163- site_url TEXT,
164- description TEXT,
165- feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),
166- last_fetched_at DATETIME,
167- last_error TEXT,
168- subscriber_count INTEGER NOT NULL DEFAULT 0,
169- etag TEXT,
170- last_modified TEXT,
171- consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
172- error_count INTEGER NOT NULL DEFAULT 0,
173- favicon_url TEXT
174- )`,
175-
176- `CREATE TABLE IF NOT EXISTS articles.subscriptions (
177- id INTEGER PRIMARY KEY AUTOINCREMENT,
178- user_did TEXT NOT NULL,
179- feed_url TEXT NOT NULL,
180- title TEXT,
181- category TEXT,
182- added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
183- uri TEXT,
184- cid TEXT,
185- UNIQUE(user_did, feed_url)
186- )`,
187-
188- `CREATE TABLE IF NOT EXISTS articles.articles (
189- id INTEGER PRIMARY KEY AUTOINCREMENT,
190- feed_url TEXT NOT NULL,
191- guid TEXT NOT NULL,
192- title TEXT NOT NULL DEFAULT '',
193- url TEXT,
194- author TEXT,
195- summary TEXT,
196- content TEXT,
197- full_content TEXT,
198- published DATETIME,
199- updated DATETIME,
200- fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
201- UNIQUE(feed_url, guid)
202- )`,
203-
204- `CREATE TABLE IF NOT EXISTS articles.read_state (
205- user_did TEXT NOT NULL,
206- article_id INTEGER NOT NULL,
207- is_read BOOLEAN NOT NULL DEFAULT 0,
208- read_at DATETIME,
209- PRIMARY KEY (user_did, article_id)
210- )`,
211-
212- `CREATE TABLE IF NOT EXISTS articles.annotations (
213- id INTEGER PRIMARY KEY AUTOINCREMENT,
214- uri TEXT NOT NULL UNIQUE,
215- author_did TEXT NOT NULL,
216- feed_url TEXT NOT NULL,
217- article_url TEXT NOT NULL,
218- quote TEXT,
219- note TEXT,
220- tags TEXT,
221- rating INTEGER,
222- created_at DATETIME NOT NULL,
223- cid TEXT
224- )`,
225-
226- `CREATE TABLE IF NOT EXISTS articles.likes (
227- id INTEGER PRIMARY KEY AUTOINCREMENT,
228- uri TEXT NOT NULL UNIQUE,
229- author_did TEXT NOT NULL,
230- feed_url TEXT NOT NULL,
231- article_url TEXT NOT NULL,
232- created_at DATETIME NOT NULL,
233- cid TEXT,
234- UNIQUE(author_did, feed_url, article_url)
235- )`,
236-
237- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`,
238- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
239- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`,
240- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`,
241- `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
242- `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`,
243- `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`,
244- `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`,
245- `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`,
246- `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`,
247- `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`,
248- `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`,
249- `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`,
250- `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`,
251- `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`,
252-
253- `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,
254- `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN
255- INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
256- END`,
257- `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN
258- INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
259- END`,
260- `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN
261- INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
262- INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
263- END`,
264-}
265-
266-var recsSchema = []string{
267- `CREATE TABLE IF NOT EXISTS recs.feed_similarity (
268- feed_a TEXT NOT NULL,
269- feed_b TEXT NOT NULL,
270- jaccard REAL NOT NULL,
271- computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
272- PRIMARY KEY (feed_a, feed_b),
273- CHECK(feed_a < feed_b)
274- )`,
275-
276- `CREATE TABLE IF NOT EXISTS recs.user_similarity (
277- user_a TEXT NOT NULL,
278- user_b TEXT NOT NULL,
279- jaccard REAL NOT NULL,
280- common_feeds INTEGER NOT NULL,
281- common_likes INTEGER NOT NULL DEFAULT 0,
282- common_tags INTEGER NOT NULL DEFAULT 0,
283- computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
284- PRIMARY KEY (user_a, user_b),
285- CHECK(user_a < user_b)
286- )`,
287-
288- `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations (
289- user_did TEXT NOT NULL,
290- target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
291- target_id TEXT NOT NULL,
292- reason TEXT,
293- dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
294- PRIMARY KEY (user_did, target_type, target_id)
295- )`,
296-
297- `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions (
298- user_did TEXT NOT NULL,
299- target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
300- target_id TEXT NOT NULL,
301- first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
302- last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
303- shown_count INTEGER NOT NULL DEFAULT 1,
304- acted BOOLEAN NOT NULL DEFAULT 0,
305- PRIMARY KEY (user_did, target_type, target_id)
306- )`,
307-
308- `CREATE TABLE IF NOT EXISTS recs.follow_distances (
309- user_a TEXT NOT NULL,
310- user_b TEXT NOT NULL,
311- distance INTEGER NOT NULL CHECK(distance IN (1, 2)),
312- PRIMARY KEY (user_a, user_b)
313- )`,
314-
315- `CREATE TABLE IF NOT EXISTS recs.user_signal_weights (
316- user_did TEXT PRIMARY KEY,
317- w_sub REAL NOT NULL DEFAULT 1.0,
318- w_like REAL NOT NULL DEFAULT 0.5,
319- w_tag REAL NOT NULL DEFAULT 0.3,
320- w_social REAL NOT NULL DEFAULT 0.7,
321- w_pop REAL NOT NULL DEFAULT 0.2,
322- w_category REAL NOT NULL DEFAULT 0.4,
323- updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
324- )`,
325-
326- `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles (
327- user_did TEXT PRIMARY KEY,
328- total_likes INTEGER NOT NULL DEFAULT 0,
329- total_tags INTEGER NOT NULL DEFAULT 0,
330- top_categories TEXT,
331- updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
332- )`,
333-
334- `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`,
335- `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`,
336- `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
337- `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`,
338- `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
339- `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`,
340- `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`,
341-}
deleted file mode 100644
@@ -1,341 +0,0 @@
1-package db
2-
3-import (
4- "database/sql"
5- "fmt"
6- "math"
7- "sync/atomic"
8- "time"
9-
10- "github.com/mattn/go-sqlite3"
11-)
12-
13-type Databases struct {
14- Users *UserStore
15- Articles *ArticleStore
16-
17- db *DB
18-}
19-
20-var multiDriverSeq int64
21-
22-func OpenAll(basePath string) (*Databases, error) {
23- articlesPath := basePath + "_articles"
24- recsPath := basePath + "_recs"
25-
26- seq := atomic.AddInt64(&multiDriverSeq, 1)
27- driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq)
28-
29- sql.Register(driverName, &sqlite3.SQLiteDriver{
30- ConnectHook: func(conn *sqlite3.SQLiteConn) error {
31- if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil {
32- return err
33- }
34- if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil {
35- return err
36- }
37- for _, p := range []string{
38- `PRAGMA wal_autocheckpoint = 1000`,
39- `PRAGMA temp_store = MEMORY`,
40- `PRAGMA mmap_size = 268435456`,
41- } {
42- if _, err := conn.Exec(p, nil); err != nil {
43- return err
44- }
45- }
46- if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil {
47- return err
48- }
49- if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil {
50- return err
51- }
52- return nil
53- },
54- })
55-
56- db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN)
57- if err != nil {
58- return nil, err
59- }
60- db.SetMaxOpenConns(10)
61- db.SetMaxIdleConns(5)
62- db.SetConnMaxLifetime(30 * time.Minute)
63- d := &DB{db}
64-
65- if err := initUsersSchema(d); err != nil {
66- d.Close()
67- return nil, err
68- }
69-
70- if err := initArticlesSchema(d); err != nil {
71- d.Close()
72- return nil, err
73- }
74-
75- if err := initRecsSchema(d); err != nil {
76- d.Close()
77- return nil, err
78- }
79-
80- return &Databases{
81- Users: NewUserStore(d),
82- Articles: NewArticleStore(d),
83- db: d,
84- }, nil
85-}
86-
87-func (d *Databases) Close() error {
88- if d.db != nil {
89- _ = d.db.Close()
90- }
91- return nil
92-}
93-
94-func (d *Databases) DB() *sql.DB {
95- return d.db.DB
96-}
97-
98-func initUsersSchema(db *DB) error {
99- for _, s := range usersSchema {
100- if _, err := db.Exec(s); err != nil {
101- return err
102- }
103- }
104- return nil
105-}
106-
107-func initArticlesSchema(db *DB) error {
108- for _, s := range articlesSchema {
109- if _, err := db.Exec(s); err != nil {
110- return err
111- }
112- }
113- return nil
114-}
115-
116-func initRecsSchema(db *DB) error {
117- for _, s := range recsSchema {
118- if _, err := db.Exec(s); err != nil {
119- return err
120- }
121- }
122- return nil
123-}
124-
125-var usersSchema = []string{
126- `CREATE TABLE IF NOT EXISTS users (
127- did TEXT PRIMARY KEY,
128- indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
129- updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
130- )`,
131-
132- `CREATE TABLE IF NOT EXISTS follows (
133- user_did TEXT NOT NULL,
134- target_did TEXT NOT NULL,
135- uri TEXT,
136- cid TEXT,
137- followed_at DATETIME,
138- PRIMARY KEY (user_did, target_did)
139- )`,
140-
141- `CREATE TABLE IF NOT EXISTS oauth_auth_requests (
142- state TEXT PRIMARY KEY,
143- data TEXT NOT NULL
144- )`,
145-
146- `CREATE TABLE IF NOT EXISTS oauth_sessions (
147- account_did TEXT NOT NULL,
148- session_id TEXT NOT NULL,
149- data TEXT NOT NULL,
150- PRIMARY KEY (account_did, session_id)
151- )`,
152-
153- `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`,
154- `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
155- `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
156- `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
157-}
158-
159-var articlesSchema = []string{
160- `CREATE TABLE IF NOT EXISTS articles.feeds (
161- feed_url TEXT PRIMARY KEY,
162- title TEXT,
163- site_url TEXT,
164- description TEXT,
165- feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),
166- last_fetched_at DATETIME,
167- last_error TEXT,
168- subscriber_count INTEGER NOT NULL DEFAULT 0,
169- etag TEXT,
170- last_modified TEXT,
171- consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
172- error_count INTEGER NOT NULL DEFAULT 0,
173- favicon_url TEXT
174- )`,
175-
176- `CREATE TABLE IF NOT EXISTS articles.subscriptions (
177- id INTEGER PRIMARY KEY AUTOINCREMENT,
178- user_did TEXT NOT NULL,
179- feed_url TEXT NOT NULL,
180- title TEXT,
181- category TEXT,
182- added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
183- uri TEXT,
184- cid TEXT,
185- UNIQUE(user_did, feed_url)
186- )`,
187-
188- `CREATE TABLE IF NOT EXISTS articles.articles (
189- id INTEGER PRIMARY KEY AUTOINCREMENT,
190- feed_url TEXT NOT NULL,
191- guid TEXT NOT NULL,
192- title TEXT NOT NULL DEFAULT '',
193- url TEXT,
194- author TEXT,
195- summary TEXT,
196- content TEXT,
197- full_content TEXT,
198- published DATETIME,
199- updated DATETIME,
200- fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
201- UNIQUE(feed_url, guid)
202- )`,
203-
204- `CREATE TABLE IF NOT EXISTS articles.read_state (
205- user_did TEXT NOT NULL,
206- article_id INTEGER NOT NULL,
207- is_read BOOLEAN NOT NULL DEFAULT 0,
208- read_at DATETIME,
209- PRIMARY KEY (user_did, article_id)
210- )`,
211-
212- `CREATE TABLE IF NOT EXISTS articles.annotations (
213- id INTEGER PRIMARY KEY AUTOINCREMENT,
214- uri TEXT NOT NULL UNIQUE,
215- author_did TEXT NOT NULL,
216- feed_url TEXT NOT NULL,
217- article_url TEXT NOT NULL,
218- quote TEXT,
219- note TEXT,
220- tags TEXT,
221- rating INTEGER,
222- created_at DATETIME NOT NULL,
223- cid TEXT
224- )`,
225-
226- `CREATE TABLE IF NOT EXISTS articles.likes (
227- id INTEGER PRIMARY KEY AUTOINCREMENT,
228- uri TEXT NOT NULL UNIQUE,
229- author_did TEXT NOT NULL,
230- feed_url TEXT NOT NULL,
231- article_url TEXT NOT NULL,
232- created_at DATETIME NOT NULL,
233- cid TEXT,
234- UNIQUE(author_did, feed_url, article_url)
235- )`,
236-
237- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`,
238- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
239- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`,
240- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`,
241- `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
242- `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`,
243- `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`,
244- `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`,
245- `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`,
246- `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`,
247- `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`,
248- `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`,
249- `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`,
250- `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`,
251- `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`,
252-
253- `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,
254- `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN
255- INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
256- END`,
257- `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN
258- INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
259- END`,
260- `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN
261- INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
262- INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
263- END`,
264-}
265-
266-var recsSchema = []string{
267- `CREATE TABLE IF NOT EXISTS recs.feed_similarity (
268- feed_a TEXT NOT NULL,
269- feed_b TEXT NOT NULL,
270- jaccard REAL NOT NULL,
271- computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
272- PRIMARY KEY (feed_a, feed_b),
273- CHECK(feed_a < feed_b)
274- )`,
275-
276- `CREATE TABLE IF NOT EXISTS recs.user_similarity (
277- user_a TEXT NOT NULL,
278- user_b TEXT NOT NULL,
279- jaccard REAL NOT NULL,
280- common_feeds INTEGER NOT NULL,
281- common_likes INTEGER NOT NULL DEFAULT 0,
282- common_tags INTEGER NOT NULL DEFAULT 0,
283- computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
284- PRIMARY KEY (user_a, user_b),
285- CHECK(user_a < user_b)
286- )`,
287-
288- `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations (
289- user_did TEXT NOT NULL,
290- target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
291- target_id TEXT NOT NULL,
292- reason TEXT,
293- dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
294- PRIMARY KEY (user_did, target_type, target_id)
295- )`,
296-
297- `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions (
298- user_did TEXT NOT NULL,
299- target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
300- target_id TEXT NOT NULL,
301- first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
302- last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
303- shown_count INTEGER NOT NULL DEFAULT 1,
304- acted BOOLEAN NOT NULL DEFAULT 0,
305- PRIMARY KEY (user_did, target_type, target_id)
306- )`,
307-
308- `CREATE TABLE IF NOT EXISTS recs.follow_distances (
309- user_a TEXT NOT NULL,
310- user_b TEXT NOT NULL,
311- distance INTEGER NOT NULL CHECK(distance IN (1, 2)),
312- PRIMARY KEY (user_a, user_b)
313- )`,
314-
315- `CREATE TABLE IF NOT EXISTS recs.user_signal_weights (
316- user_did TEXT PRIMARY KEY,
317- w_sub REAL NOT NULL DEFAULT 1.0,
318- w_like REAL NOT NULL DEFAULT 0.5,
319- w_tag REAL NOT NULL DEFAULT 0.3,
320- w_social REAL NOT NULL DEFAULT 0.7,
321- w_pop REAL NOT NULL DEFAULT 0.2,
322- w_category REAL NOT NULL DEFAULT 0.4,
323- updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
324- )`,
325-
326- `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles (
327- user_did TEXT PRIMARY KEY,
328- total_likes INTEGER NOT NULL DEFAULT 0,
329- total_tags INTEGER NOT NULL DEFAULT 0,
330- top_categories TEXT,
331- updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
332- )`,
333-
334- `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`,
335- `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`,
336- `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
337- `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`,
338- `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
339- `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`,
340- `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`,
341-}
modified internal/db/social.go +1 -1
@@ -240,7 +240,7 @@ func (s *ArticleStore) GetLikeCount(ctx context.Context, feedURL, articleURL str
240240 func (s *ArticleStore) GetLike(ctx context.Context, authorDID, feedURL, articleURL string) (*Like, error) {
241241 l := &Like{}
242242 err := s.db.QueryRowContext(ctx, `
243- SELECT id, uri, author_did, feed_url, article_url, created_at, cid FROM articles.likes
243+ SELECT * FROM articles.likes
244244 WHERE author_did = ? AND feed_url = ? AND article_url = ?
245245 `, authorDID, feedURL, articleURL).Scan(&l.ID, &l.URI, &l.AuthorDID, &l.FeedURL, &l.ArticleURL, &l.CreatedAt, &l.CID)
246246 if err != nil {
@@ -240,7 +240,7 @@ func (s *ArticleStore) GetLikeCount(ctx context.Context, feedURL, articleURL str
240 func (s *ArticleStore) GetLike(ctx context.Context, authorDID, feedURL, articleURL string) (*Like, error) {240 func (s *ArticleStore) GetLike(ctx context.Context, authorDID, feedURL, articleURL string) (*Like, error) {
241 l := &Like{}241 l := &Like{}
242 err := s.db.QueryRowContext(ctx, `242 err := s.db.QueryRowContext(ctx, `
243- SELECT id, uri, author_did, feed_url, article_url, created_at, cid FROM articles.likes243+ SELECT * FROM articles.likes
244 WHERE author_did = ? AND feed_url = ? AND article_url = ?244 WHERE author_did = ? AND feed_url = ? AND article_url = ?
245 `, authorDID, feedURL, articleURL).Scan(&l.ID, &l.URI, &l.AuthorDID, &l.FeedURL, &l.ArticleURL, &l.CreatedAt, &l.CID)245 `, authorDID, feedURL, articleURL).Scan(&l.ID, &l.URI, &l.AuthorDID, &l.FeedURL, &l.ArticleURL, &l.CreatedAt, &l.CID)
246 if err != nil {246 if err != nil {
modified internal/db/user.go +1 -2
@@ -60,8 +60,7 @@ func (s *UserStore) CreateUser(ctx context.Context, did string) (*User, error) {
6060 func (s *UserStore) GetUser(ctx context.Context, did string) (*User, error) {
6161 u := &User{}
6262 err := s.db.QueryRowContext(ctx, `
63- SELECT did, indexed_at, updated_at
64- FROM users WHERE did = ?
63+ SELECT * FROM users WHERE did = ?
6564 `, did).Scan(&u.DID, &u.IndexedAt, &u.UpdatedAt)
6665 if err != nil {
6766 return nil, err
@@ -60,8 +60,7 @@ func (s *UserStore) CreateUser(ctx context.Context, did string) (*User, error) {
60 func (s *UserStore) GetUser(ctx context.Context, did string) (*User, error) {60 func (s *UserStore) GetUser(ctx context.Context, did string) (*User, error) {
61 u := &User{}61 u := &User{}
62 err := s.db.QueryRowContext(ctx, `62 err := s.db.QueryRowContext(ctx, `
63- SELECT did, indexed_at, updated_at63+ SELECT * FROM users WHERE did = ?
64- FROM users WHERE did = ?
65 `, did).Scan(&u.DID, &u.IndexedAt, &u.UpdatedAt)64 `, did).Scan(&u.DID, &u.IndexedAt, &u.UpdatedAt)
66 if err != nil {65 if err != nil {
67 return nil, err66 return nil, err
modified internal/feed/discover.go +2 -2
@@ -130,7 +130,7 @@ func findFavicon(ctx context.Context, base *url.URL, links []string) string {
130130 if err != nil {
131131 return
132132 }
133- resp.Body.Close()
133+ _ = resp.Body.Close()
134134 if resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type")) {
135135 select {
136136 case found <- result{url: cleanFavicon(resolved.String()), found: true}:
@@ -160,7 +160,7 @@ func checkContentType(ctx context.Context, url string) bool {
160160 if err != nil {
161161 return false
162162 }
163- resp.Body.Close()
163+ _ = resp.Body.Close()
164164 return resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type"))
165165 }
166166
@@ -130,7 +130,7 @@ func findFavicon(ctx context.Context, base *url.URL, links []string) string {
130 if err != nil {130 if err != nil {
131 return131 return
132 }132 }
133- resp.Body.Close()133+ _ = resp.Body.Close()
134 if resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type")) {134 if resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type")) {
135 select {135 select {
136 case found <- result{url: cleanFavicon(resolved.String()), found: true}:136 case found <- result{url: cleanFavicon(resolved.String()), found: true}:
@@ -160,7 +160,7 @@ func checkContentType(ctx context.Context, url string) bool {
160 if err != nil {160 if err != nil {
161 return false161 return false
162 }162 }
163- resp.Body.Close()163+ _ = resp.Body.Close()
164 return resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type"))164 return resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type"))
165 }165 }
166 166
modified internal/feed/fetcher.go +1 -1
@@ -221,7 +221,7 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) {
221221 metrics.FeedsFetchedLast.Set(float64(time.Now().Unix()))
222222 if err != nil {
223223 s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL)
224- s.store.RecordFetchError(ctx, feed.URL, err.Error())
224+ _ = s.store.RecordFetchError(ctx, feed.URL, err.Error())
225225 return
226226 }
227227
@@ -221,7 +221,7 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) {
221 metrics.FeedsFetchedLast.Set(float64(time.Now().Unix()))221 metrics.FeedsFetchedLast.Set(float64(time.Now().Unix()))
222 if err != nil {222 if err != nil {
223 s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL)223 s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL)
224- s.store.RecordFetchError(ctx, feed.URL, err.Error())224+ _ = s.store.RecordFetchError(ctx, feed.URL, err.Error())
225 return225 return
226 }226 }
227 227
modified internal/httpclient/httpclient.go +1 -1
@@ -19,7 +19,7 @@ const (
1919
2020 var dnsResolver = &net.Resolver{
2121 PreferGo: true,
22- Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
22+ Dial: func(ctx context.Context, _, address string) (net.Conn, error) {
2323 d := net.Dialer{Timeout: 3 * time.Second}
2424 return d.DialContext(ctx, "udp", "1.1.1.1:53")
2525 },
@@ -19,7 +19,7 @@ const (
19 19
20 var dnsResolver = &net.Resolver{20 var dnsResolver = &net.Resolver{
21 PreferGo: true,21 PreferGo: true,
22- Dial: func(ctx context.Context, network, address string) (net.Conn, error) {22+ Dial: func(ctx context.Context, _, address string) (net.Conn, error) {
23 d := net.Dialer{Timeout: 3 * time.Second}23 d := net.Dialer{Timeout: 3 * time.Second}
24 return d.DialContext(ctx, "udp", "1.1.1.1:53")24 return d.DialContext(ctx, "udp", "1.1.1.1:53")
25 },25 },
modified internal/server/articles_handler.go +1 -1
@@ -104,7 +104,7 @@ func (s *Server) handleArticles(w http.ResponseWriter, r *http.Request) {
104104 }
105105 }
106106
107- if r.Header.Get("HX-Request") == "true" {
107+ if r.Header.Get("HX-Request") == htmxRequestHeader {
108108 s.render(w, r, "articles-content.html", data)
109109 return
110110 }
@@ -104,7 +104,7 @@ func (s *Server) handleArticles(w http.ResponseWriter, r *http.Request) {
104 }104 }
105 }105 }
106 106
107- if r.Header.Get("HX-Request") == "true" {107+ if r.Header.Get("HX-Request") == htmxRequestHeader {
108 s.render(w, r, "articles-content.html", data)108 s.render(w, r, "articles-content.html", data)
109 return109 return
110 }110 }
modified internal/server/middleware.go +1 -1
@@ -55,7 +55,7 @@ func (s *Server) csrfMiddleware(next http.Handler) http.Handler {
5555 return
5656 }
5757
58- if r.Header.Get("HX-Request") == "true" {
58+ if r.Header.Get("HX-Request") == htmxRequestHeader {
5959 origin := r.Header.Get("Origin")
6060 if origin != "" && !sameOrigin(origin, r.Host) {
6161 http.Error(w, "forbidden", http.StatusForbidden)
@@ -55,7 +55,7 @@ func (s *Server) csrfMiddleware(next http.Handler) http.Handler {
55 return55 return
56 }56 }
57 57
58- if r.Header.Get("HX-Request") == "true" {58+ if r.Header.Get("HX-Request") == htmxRequestHeader {
59 origin := r.Header.Get("Origin")59 origin := r.Header.Get("Origin")
60 if origin != "" && !sameOrigin(origin, r.Host) {60 if origin != "" && !sameOrigin(origin, r.Host) {
61 http.Error(w, "forbidden", http.StatusForbidden)61 http.Error(w, "forbidden", http.StatusForbidden)
modified internal/server/server.go +6 -8
@@ -33,6 +33,8 @@ import (
3333 "pkg.rbrt.fr/glean/static"
3434 )
3535
36+const htmxRequestHeader = "true"
37+
3638 var oauthScopes = []string{
3739 "atproto",
3840 "blob:*/*",
@@ -314,12 +316,8 @@ func (s *Server) loadTemplates() {
314316 "sanitizeHTML": func(input string) template.HTML {
315317 return template.HTML(sanitize.HTML(input))
316318 },
317- "plainText": func(input string) string {
318- return sanitize.PlainText(input)
319- },
320- "now": func() time.Time {
321- return time.Now()
322- },
319+ "plainText": sanitize.PlainText,
320+ "now": time.Now,
323321 "activeClass": func(activePath, linkPath string) string {
324322 if activePath == linkPath || (len(activePath) > len(linkPath) && activePath[:len(linkPath)+1] == linkPath+"/") {
325323 return "bg-spot-hover text-spot-text font-bold"
@@ -515,7 +513,7 @@ func (s *Server) handleNotFound(w http.ResponseWriter, r *http.Request) {
515513 }
516514
517515 func (s *Server) renderError(w http.ResponseWriter, r *http.Request, code int, title, message string) {
518- if r.Header.Get("HX-Request") == "true" {
516+ if r.Header.Get("HX-Request") == htmxRequestHeader {
519517 w.WriteHeader(code)
520518 w.Write([]byte(message))
521519 return
@@ -536,7 +534,7 @@ func (s *Server) render(w http.ResponseWriter, r *http.Request, name string, dat
536534 data["CSRFToken"] = cookie.Value
537535 }
538536
539- if r.Header.Get("HX-Request") == "true" {
537+ if r.Header.Get("HX-Request") == htmxRequestHeader {
540538 if err := s.templates.ExecuteTemplate(w, name, data); err != nil {
541539 s.logger.Error("template error", "error", err, "template", name)
542540 http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -33,6 +33,8 @@ import (
33 "pkg.rbrt.fr/glean/static"33 "pkg.rbrt.fr/glean/static"
34 )34 )
35 35
36+const htmxRequestHeader = "true"
37+
36 var oauthScopes = []string{38 var oauthScopes = []string{
37 "atproto",39 "atproto",
38 "blob:*/*",40 "blob:*/*",
@@ -314,12 +316,8 @@ func (s *Server) loadTemplates() {
314 "sanitizeHTML": func(input string) template.HTML {316 "sanitizeHTML": func(input string) template.HTML {
315 return template.HTML(sanitize.HTML(input))317 return template.HTML(sanitize.HTML(input))
316 },318 },
317- "plainText": func(input string) string {319+ "plainText": sanitize.PlainText,
318- return sanitize.PlainText(input)320+ "now": time.Now,
319- },
320- "now": func() time.Time {
321- return time.Now()
322- },
323 "activeClass": func(activePath, linkPath string) string {321 "activeClass": func(activePath, linkPath string) string {
324 if activePath == linkPath || (len(activePath) > len(linkPath) && activePath[:len(linkPath)+1] == linkPath+"/") {322 if activePath == linkPath || (len(activePath) > len(linkPath) && activePath[:len(linkPath)+1] == linkPath+"/") {
325 return "bg-spot-hover text-spot-text font-bold"323 return "bg-spot-hover text-spot-text font-bold"
@@ -515,7 +513,7 @@ func (s *Server) handleNotFound(w http.ResponseWriter, r *http.Request) {
515 }513 }
516 514
517 func (s *Server) renderError(w http.ResponseWriter, r *http.Request, code int, title, message string) {515 func (s *Server) renderError(w http.ResponseWriter, r *http.Request, code int, title, message string) {
518- if r.Header.Get("HX-Request") == "true" {516+ if r.Header.Get("HX-Request") == htmxRequestHeader {
519 w.WriteHeader(code)517 w.WriteHeader(code)
520 w.Write([]byte(message))518 w.Write([]byte(message))
521 return519 return
@@ -536,7 +534,7 @@ func (s *Server) render(w http.ResponseWriter, r *http.Request, name string, dat
536 data["CSRFToken"] = cookie.Value534 data["CSRFToken"] = cookie.Value
537 }535 }
538 536
539- if r.Header.Get("HX-Request") == "true" {537+ if r.Header.Get("HX-Request") == htmxRequestHeader {
540 if err := s.templates.ExecuteTemplate(w, name, data); err != nil {538 if err := s.templates.ExecuteTemplate(w, name, data); err != nil {
541 s.logger.Error("template error", "error", err, "template", name)539 s.logger.Error("template error", "error", err, "template", name)
542 http.Error(w, err.Error(), http.StatusInternalServerError)540 http.Error(w, err.Error(), http.StatusInternalServerError)