rickub
nandi/gleanpublic⑂ Fork 0
⑂ 926777b
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Add batch sync for subscriptions, likes, and annotationsUnverified

Julien Robert committed 2026-04-23T11:53:55+02:00 Browse files
926777b parent: 7741164
modified go.mod +1 -0
@@ -42,6 +42,7 @@ require (
4242 gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
4343 gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
4444 golang.org/x/crypto v0.50.0 // indirect
45+ golang.org/x/sync v0.20.0 // indirect
4546 golang.org/x/sys v0.43.0 // indirect
4647 golang.org/x/text v0.36.0 // indirect
4748 golang.org/x/time v0.5.0 // indirect
@@ -42,6 +42,7 @@ require (
42 gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect42 gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
43 gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect43 gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
44 golang.org/x/crypto v0.50.0 // indirect44 golang.org/x/crypto v0.50.0 // indirect
45+ golang.org/x/sync v0.20.0 // indirect
45 golang.org/x/sys v0.43.0 // indirect46 golang.org/x/sys v0.43.0 // indirect
46 golang.org/x/text v0.36.0 // indirect47 golang.org/x/text v0.36.0 // indirect
47 golang.org/x/time v0.5.0 // indirect48 golang.org/x/time v0.5.0 // indirect
modified go.sum +2 -0
@@ -75,6 +75,8 @@ golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
7575 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
7676 golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
7777 golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
78+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
79+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
7880 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
7981 golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
8082 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
@@ -75,6 +75,8 @@ golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
75 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=75 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
76 golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=76 golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
77 golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=77 golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
78+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
79+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
78 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=80 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
79 golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=81 golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
80 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=82 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
modified internal/atproto/sync.go +169 -162
@@ -5,17 +5,13 @@
55 // - we only sync known users (not the entire network)
66 // - the Jetstream consumer handles real-time events concurrently
77 // - all reconcile operations are idempotent
8-//
9-// Known trade-off: syncFollows atomically replaces all follows for a user.
10-// A Jetstream follow event arriving mid-sync could be lost, but self-heals
11-// on the next sync cycle or Jetstream event.
128 package atproto
139
1410 import (
1511 "context"
1612 "encoding/json"
17- "errors"
1813 "log/slog"
14+ "sync"
1915 "time"
2016
2117 "pkg.rbrt.fr/glean/internal/db"
@@ -35,19 +31,19 @@ func NewSync(articles, users *db.DB, client *Client, logger *slog.Logger) *Sync
3531 func (s *Sync) Run(ctx context.Context, userDID string) error {
3632 s.logger.Info("syncing from PDS", "did", userDID)
3733
38- if err := s.syncCollection(ctx, userDID, CollectionSubscription, s.reconcileSubscription); err != nil {
34+ if err := s.syncCollection(ctx, userDID, CollectionSubscription, s.batchReconcileSubscriptions); err != nil {
3935 s.logger.Error("sync subscriptions failed", "error", err, "did", userDID)
4036 }
41- if err := s.syncCollection(ctx, userDID, CollectionSkyreaderSubscription, s.reconcileSkyreaderSubscription); err != nil {
37+ if err := s.syncCollection(ctx, userDID, CollectionSkyreaderSubscription, s.batchReconcileSkyreaderSubscriptions); err != nil {
4238 s.logger.Error("sync skyreader subscriptions failed", "error", err, "did", userDID)
4339 }
44- if err := s.syncCollection(ctx, userDID, CollectionLike, s.reconcileLike); err != nil {
40+ if err := s.syncCollection(ctx, userDID, CollectionLike, s.batchReconcileLikes); err != nil {
4541 s.logger.Error("sync likes failed", "error", err, "did", userDID)
4642 }
47- if err := s.syncCollection(ctx, userDID, CollectionAnnotation, s.reconcileAnnotation); err != nil {
43+ if err := s.syncCollection(ctx, userDID, CollectionAnnotation, s.batchReconcileAnnotations); err != nil {
4844 s.logger.Error("sync annotations failed", "error", err, "did", userDID)
4945 }
50- if err := s.syncCollection(ctx, userDID, CollectionMarginNote, s.reconcileMarginNote); err != nil {
46+ if err := s.syncCollection(ctx, userDID, CollectionMarginNote, s.batchReconcileMarginNotes); err != nil {
5147 s.logger.Error("sync margin notes failed", "error", err, "did", userDID)
5248 }
5349 if err := s.syncFollows(ctx, userDID); err != nil {
@@ -57,187 +53,171 @@ func (s *Sync) Run(ctx context.Context, userDID string) error {
5753 return nil
5854 }
5955
60-type reconcileFunc func(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error
61-
62-func (s *Sync) syncCollection(ctx context.Context, userDID, collection string, fn reconcileFunc) error {
56+func (s *Sync) syncCollection(ctx context.Context, userDID, collection string, fn func(ctx context.Context, userDID string, records []Record) error) error {
57+ var allRecords []Record
6358 cursor := ""
6459 for {
6560 records, next, err := s.client.ListRecords(ctx, userDID, collection, 100, cursor)
6661 if err != nil {
6762 return err
6863 }
69-
70- for _, r := range records {
71- if err := fn(ctx, userDID, r.URI, r.CID, r.Value); err != nil {
72- s.logger.Error("reconcile record error", "error", err, "uri", r.URI)
73- }
74- }
75-
64+ allRecords = append(allRecords, records...)
7665 if next == "" || len(records) == 0 {
7766 break
7867 }
7968 cursor = next
8069 }
81- return nil
82-}
83-
84-func (s *Sync) reconcileSubscription(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {
85- var rec SubscriptionRecord
86- if err := json.Unmarshal(value, &rec); err != nil {
87- return err
88- }
89-
90- if rec.FeedURL == "" {
70+ if len(allRecords) == 0 {
9171 return nil
9272 }
73+ return fn(ctx, userDID, allRecords)
74+}
9375
94- f := &db.Feed{FeedURL: rec.FeedURL, Title: db.NullStr(rec.Title)}
95- _ = s.articles.UpsertFeed(ctx, f)
76+func (s *Sync) batchReconcileSubscriptions(ctx context.Context, userDID string, records []Record) error {
77+ var feeds []*db.Feed
78+ var subs []db.SubData
9679
97- existing, err := s.articles.GetSubscription(ctx, userDID, rec.FeedURL)
98- if err == nil && existing != nil {
99- if !existing.URI.Valid || existing.URI.String == "" {
100- return s.articles.UpdateSubscriptionURI(ctx, userDID, rec.FeedURL, uri, cid)
80+ for _, r := range records {
81+ var rec SubscriptionRecord
82+ if err := json.Unmarshal(r.Value, &rec); err != nil {
83+ continue
10184 }
102- return nil
85+ if rec.FeedURL == "" {
86+ continue
87+ }
88+ feeds = append(feeds, &db.Feed{FeedURL: rec.FeedURL, Title: db.NullStr(rec.Title)})
89+ subs = append(subs, db.SubData{
90+ FeedURL: rec.FeedURL,
91+ Title: rec.Title,
92+ Category: rec.Category,
93+ URI: r.URI,
94+ CID: r.CID,
95+ })
10396 }
10497
105- err = s.articles.CreateSubscription(ctx, userDID, rec.FeedURL, rec.Title, rec.Category, uri, cid)
106- if errors.Is(err, db.ErrDuplicateSubscription) {
107- return nil
98+ if len(feeds) > 0 {
99+ _ = s.articles.BatchUpsertFeeds(ctx, feeds)
108100 }
109- return err
101+ return s.articles.BatchReconcileSubscriptions(ctx, userDID, subs)
110102 }
111103
112-func (s *Sync) reconcileSkyreaderSubscription(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {
113- var rec SkyreaderSubscriptionRecord
114- if err := json.Unmarshal(value, &rec); err != nil {
115- return err
116- }
117-
118- if rec.FeedURL == "" {
119- return nil
120- }
104+func (s *Sync) batchReconcileSkyreaderSubscriptions(ctx context.Context, userDID string, records []Record) error {
105+ var feeds []*db.Feed
106+ var subs []db.SubData
121107
122- f := &db.Feed{FeedURL: rec.FeedURL, Title: db.NullStr(rec.Title), SiteURL: db.NullStr(rec.SiteURL)}
123- _ = s.articles.UpsertFeed(ctx, f)
124-
125- existing, err := s.articles.GetSubscription(ctx, userDID, rec.FeedURL)
126- if err == nil && existing != nil {
127- if !existing.URI.Valid || existing.URI.String == "" {
128- return s.articles.UpdateSubscriptionURI(ctx, userDID, rec.FeedURL, uri, cid)
108+ for _, r := range records {
109+ var rec SkyreaderSubscriptionRecord
110+ if err := json.Unmarshal(r.Value, &rec); err != nil {
111+ continue
129112 }
130- return nil
113+ if rec.FeedURL == "" {
114+ continue
115+ }
116+ feeds = append(feeds, &db.Feed{FeedURL: rec.FeedURL, Title: db.NullStr(rec.Title), SiteURL: db.NullStr(rec.SiteURL)})
117+ subs = append(subs, db.SubData{
118+ FeedURL: rec.FeedURL,
119+ Title: rec.Title,
120+ URI: r.URI,
121+ CID: r.CID,
122+ })
131123 }
132124
133- err = s.articles.CreateSubscription(ctx, userDID, rec.FeedURL, rec.Title, "", uri, cid)
134- if errors.Is(err, db.ErrDuplicateSubscription) {
135- return nil
125+ if len(feeds) > 0 {
126+ _ = s.articles.BatchUpsertFeeds(ctx, feeds)
136127 }
137- return err
128+ return s.articles.BatchReconcileSubscriptions(ctx, userDID, subs)
138129 }
139130
140-func (s *Sync) reconcileLike(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {
141- var rec LikeRecord
142- if err := json.Unmarshal(value, &rec); err != nil {
143- return err
144- }
145-
146- if rec.FeedURL == "" || rec.ArticleURL == "" {
147- return nil
148- }
149-
150- exists, err := s.articles.HasLiked(ctx, userDID, rec.FeedURL, rec.ArticleURL)
151- if err != nil {
152- return err
153- }
154- if exists {
155- return nil
156- }
131+func (s *Sync) batchReconcileLikes(ctx context.Context, userDID string, records []Record) error {
132+ var likes []*db.Like
157133
158- t, _ := time.Parse(time.RFC3339, rec.CreatedAt)
159- like := &db.Like{
160- URI: uri,
161- AuthorDID: userDID,
162- FeedURL: rec.FeedURL,
163- ArticleURL: rec.ArticleURL,
164- CreatedAt: db.NullTime(t),
165- CID: db.NullStr(cid),
166- }
167- err = s.articles.CreateLike(ctx, like)
168- if errors.Is(err, db.ErrDuplicateLike) {
169- return nil
170- }
171- return err
134+ for _, r := range records {
135+ var rec LikeRecord
136+ if err := json.Unmarshal(r.Value, &rec); err != nil {
137+ continue
138+ }
139+ if rec.FeedURL == "" || rec.ArticleURL == "" {
140+ continue
141+ }
142+ t, _ := time.Parse(time.RFC3339, rec.CreatedAt)
143+ likes = append(likes, &db.Like{
144+ URI: r.URI,
145+ AuthorDID: userDID,
146+ FeedURL: rec.FeedURL,
147+ ArticleURL: rec.ArticleURL,
148+ CreatedAt: db.NullTime(t),
149+ CID: db.NullStr(r.CID),
150+ })
151+ }
152+
153+ return s.articles.BatchCreateLikes(ctx, likes)
172154 }
173155
174-func (s *Sync) reconcileAnnotation(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {
175- var rec AnnotationRecord
176- if err := json.Unmarshal(value, &rec); err != nil {
177- return err
178- }
179-
180- if rec.FeedURL == "" || rec.ArticleURL == "" {
181- return nil
182- }
156+func (s *Sync) batchReconcileAnnotations(ctx context.Context, userDID string, records []Record) error {
157+ var annotations []*db.Annotation
183158
184- exists, err := s.articles.AnnotationExists(ctx, uri)
185- if err != nil || exists {
186- return err
159+ for _, r := range records {
160+ var rec AnnotationRecord
161+ if err := json.Unmarshal(r.Value, &rec); err != nil {
162+ continue
163+ }
164+ if rec.FeedURL == "" || rec.ArticleURL == "" {
165+ continue
166+ }
167+ t, _ := time.Parse(time.RFC3339, rec.CreatedAt)
168+ a := &db.Annotation{
169+ URI: r.URI,
170+ AuthorDID: userDID,
171+ FeedURL: rec.FeedURL,
172+ ArticleURL: rec.ArticleURL,
173+ Quote: db.NullStr(rec.Quote),
174+ Note: db.NullStr(rec.Note),
175+ Tags: db.NullStrTags(rec.Tags),
176+ CreatedAt: db.NullTime(t),
177+ CID: db.NullStr(r.CID),
178+ }
179+ if rec.Rating > 0 {
180+ a.Rating = db.NullInt(int64(rec.Rating))
181+ }
182+ annotations = append(annotations, a)
187183 }
188184
189- t, _ := time.Parse(time.RFC3339, rec.CreatedAt)
190- a := &db.Annotation{
191- URI: uri,
192- AuthorDID: userDID,
193- FeedURL: rec.FeedURL,
194- ArticleURL: rec.ArticleURL,
195- Quote: db.NullStr(rec.Quote),
196- Note: db.NullStr(rec.Note),
197- Tags: db.NullStrTags(rec.Tags),
198- CreatedAt: db.NullTime(t),
199- CID: db.NullStr(cid),
200- }
201- if rec.Rating > 0 {
202- a.Rating = db.NullInt(int64(rec.Rating))
203- }
204- return s.articles.CreateAnnotation(ctx, a)
185+ return s.articles.BatchCreateAnnotations(ctx, annotations)
205186 }
206187
207-func (s *Sync) reconcileMarginNote(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {
208- var rec MarginNoteRecord
209- if err := json.Unmarshal(value, &rec); err != nil {
210- return err
211- }
188+func (s *Sync) batchReconcileMarginNotes(ctx context.Context, userDID string, records []Record) error {
189+ var annotations []*db.Annotation
212190
213- articleURL, quote, note, tags := rec.ToAnnotation()
214- if articleURL == "" {
215- return nil
216- }
217-
218- exists, err := s.articles.AnnotationExists(ctx, uri)
219- if err != nil || exists {
220- return err
221- }
191+ for _, r := range records {
192+ var rec MarginNoteRecord
193+ if err := json.Unmarshal(r.Value, &rec); err != nil {
194+ continue
195+ }
196+ articleURL, quote, note, tags := rec.ToAnnotation()
197+ if articleURL == "" {
198+ continue
199+ }
222200
223- feedURL := ""
224- if article, err := s.articles.GetArticleByURL(ctx, articleURL); err == nil {
225- feedURL = article.FeedURL
226- }
201+ feedURL := ""
202+ if article, err := s.articles.GetArticleByURL(ctx, articleURL); err == nil {
203+ feedURL = article.FeedURL
204+ }
227205
228- t, _ := time.Parse(time.RFC3339, rec.CreatedAt)
229- a := &db.Annotation{
230- URI: uri,
231- AuthorDID: userDID,
232- FeedURL: feedURL,
233- ArticleURL: articleURL,
234- Quote: db.NullStr(quote),
235- Note: db.NullStr(note),
236- Tags: db.NullStrTags(tags),
237- CreatedAt: db.NullTime(t),
238- CID: db.NullStr(cid),
239- }
240- return s.articles.CreateAnnotation(ctx, a)
206+ t, _ := time.Parse(time.RFC3339, rec.CreatedAt)
207+ annotations = append(annotations, &db.Annotation{
208+ URI: r.URI,
209+ AuthorDID: userDID,
210+ FeedURL: feedURL,
211+ ArticleURL: articleURL,
212+ Quote: db.NullStr(quote),
213+ Note: db.NullStr(note),
214+ Tags: db.NullStrTags(tags),
215+ CreatedAt: db.NullTime(t),
216+ CID: db.NullStr(r.CID),
217+ })
218+ }
219+
220+ return s.articles.BatchCreateAnnotations(ctx, annotations)
241221 }
242222
243223 func (s *Sync) syncFollows(ctx context.Context, userDID string) error {
@@ -266,16 +246,6 @@ func (s *Sync) syncFollows(ctx context.Context, userDID string) error {
266246 CID: db.NullStr(r.CID),
267247 FollowedAt: db.NullTime(t),
268248 }
269-
270- // auto onboard followers
271- var handle, displayName, avatarURL string
272- if h, dn, avatar, err := FetchProfile(ctx, rec.Subject); err == nil {
273- handle = h
274- displayName = dn
275- avatarURL = avatar
276- }
277-
278- s.users.CreateUser(ctx, rec.Subject, handle, displayName, avatarURL)
279249 }
280250
281251 if next == "" || len(records) == 0 {
@@ -289,5 +259,42 @@ func (s *Sync) syncFollows(ctx context.Context, userDID string) error {
289259 return nil
290260 }
291261
262+ type profileResult struct {
263+ did string
264+ handle string
265+ displayName string
266+ avatarURL string
267+ }
268+
269+ var wg sync.WaitGroup
270+ sem := make(chan struct{}, 10)
271+ results := make([]profileResult, 0, len(activeFollows))
272+ var mu sync.Mutex
273+
274+ for targetDID := range activeFollows {
275+ sem <- struct{}{}
276+ wg.Add(1)
277+ go func(did string) {
278+ defer func() { <-sem }()
279+ defer wg.Done()
280+ var handle, displayName, avatarURL string
281+ if h, dn, avatar, err := FetchProfile(ctx, did); err == nil {
282+ handle = h
283+ displayName = dn
284+ avatarURL = avatar
285+ }
286+ mu.Lock()
287+ results = append(results, profileResult{did, handle, displayName, avatarURL})
288+ mu.Unlock()
289+ }(targetDID)
290+ }
291+ wg.Wait()
292+
293+ var users []db.UserData
294+ for _, r := range results {
295+ users = append(users, db.UserData{DID: r.did, Handle: r.handle, DisplayName: r.displayName, AvatarURL: r.avatarURL})
296+ }
297+ _ = s.users.BatchCreateUsers(ctx, users)
298+
292299 return s.users.SyncFollows(ctx, userDID, activeFollows)
293300 }
@@ -5,17 +5,13 @@
5 // - we only sync known users (not the entire network)5 // - we only sync known users (not the entire network)
6 // - the Jetstream consumer handles real-time events concurrently6 // - the Jetstream consumer handles real-time events concurrently
7 // - all reconcile operations are idempotent7 // - all reconcile operations are idempotent
8-//
9-// Known trade-off: syncFollows atomically replaces all follows for a user.
10-// A Jetstream follow event arriving mid-sync could be lost, but self-heals
11-// on the next sync cycle or Jetstream event.
12 package atproto8 package atproto
13 9
14 import (10 import (
15 "context"11 "context"
16 "encoding/json"12 "encoding/json"
17- "errors"
18 "log/slog"13 "log/slog"
14+ "sync"
19 "time"15 "time"
20 16
21 "pkg.rbrt.fr/glean/internal/db"17 "pkg.rbrt.fr/glean/internal/db"
@@ -35,19 +31,19 @@ func NewSync(articles, users *db.DB, client *Client, logger *slog.Logger) *Sync
35 func (s *Sync) Run(ctx context.Context, userDID string) error {31 func (s *Sync) Run(ctx context.Context, userDID string) error {
36 s.logger.Info("syncing from PDS", "did", userDID)32 s.logger.Info("syncing from PDS", "did", userDID)
37 33
38- if err := s.syncCollection(ctx, userDID, CollectionSubscription, s.reconcileSubscription); err != nil {34+ if err := s.syncCollection(ctx, userDID, CollectionSubscription, s.batchReconcileSubscriptions); err != nil {
39 s.logger.Error("sync subscriptions failed", "error", err, "did", userDID)35 s.logger.Error("sync subscriptions failed", "error", err, "did", userDID)
40 }36 }
41- if err := s.syncCollection(ctx, userDID, CollectionSkyreaderSubscription, s.reconcileSkyreaderSubscription); err != nil {37+ if err := s.syncCollection(ctx, userDID, CollectionSkyreaderSubscription, s.batchReconcileSkyreaderSubscriptions); err != nil {
42 s.logger.Error("sync skyreader subscriptions failed", "error", err, "did", userDID)38 s.logger.Error("sync skyreader subscriptions failed", "error", err, "did", userDID)
43 }39 }
44- if err := s.syncCollection(ctx, userDID, CollectionLike, s.reconcileLike); err != nil {40+ if err := s.syncCollection(ctx, userDID, CollectionLike, s.batchReconcileLikes); err != nil {
45 s.logger.Error("sync likes failed", "error", err, "did", userDID)41 s.logger.Error("sync likes failed", "error", err, "did", userDID)
46 }42 }
47- if err := s.syncCollection(ctx, userDID, CollectionAnnotation, s.reconcileAnnotation); err != nil {43+ if err := s.syncCollection(ctx, userDID, CollectionAnnotation, s.batchReconcileAnnotations); err != nil {
48 s.logger.Error("sync annotations failed", "error", err, "did", userDID)44 s.logger.Error("sync annotations failed", "error", err, "did", userDID)
49 }45 }
50- if err := s.syncCollection(ctx, userDID, CollectionMarginNote, s.reconcileMarginNote); err != nil {46+ if err := s.syncCollection(ctx, userDID, CollectionMarginNote, s.batchReconcileMarginNotes); err != nil {
51 s.logger.Error("sync margin notes failed", "error", err, "did", userDID)47 s.logger.Error("sync margin notes failed", "error", err, "did", userDID)
52 }48 }
53 if err := s.syncFollows(ctx, userDID); err != nil {49 if err := s.syncFollows(ctx, userDID); err != nil {
@@ -57,187 +53,171 @@ func (s *Sync) Run(ctx context.Context, userDID string) error {
57 return nil53 return nil
58 }54 }
59 55
60-type reconcileFunc func(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error56+func (s *Sync) syncCollection(ctx context.Context, userDID, collection string, fn func(ctx context.Context, userDID string, records []Record) error) error {
61-57+ var allRecords []Record
62-func (s *Sync) syncCollection(ctx context.Context, userDID, collection string, fn reconcileFunc) error {
63 cursor := ""58 cursor := ""
64 for {59 for {
65 records, next, err := s.client.ListRecords(ctx, userDID, collection, 100, cursor)60 records, next, err := s.client.ListRecords(ctx, userDID, collection, 100, cursor)
66 if err != nil {61 if err != nil {
67 return err62 return err
68 }63 }
69-64+ allRecords = append(allRecords, records...)
70- for _, r := range records {
71- if err := fn(ctx, userDID, r.URI, r.CID, r.Value); err != nil {
72- s.logger.Error("reconcile record error", "error", err, "uri", r.URI)
73- }
74- }
75-
76 if next == "" || len(records) == 0 {65 if next == "" || len(records) == 0 {
77 break66 break
78 }67 }
79 cursor = next68 cursor = next
80 }69 }
81- return nil70+ if len(allRecords) == 0 {
82-}
83-
84-func (s *Sync) reconcileSubscription(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {
85- var rec SubscriptionRecord
86- if err := json.Unmarshal(value, &rec); err != nil {
87- return err
88- }
89-
90- if rec.FeedURL == "" {
91 return nil71 return nil
92 }72 }
73+ return fn(ctx, userDID, allRecords)
74+}
93 75
94- f := &db.Feed{FeedURL: rec.FeedURL, Title: db.NullStr(rec.Title)}76+func (s *Sync) batchReconcileSubscriptions(ctx context.Context, userDID string, records []Record) error {
95- _ = s.articles.UpsertFeed(ctx, f)77+ var feeds []*db.Feed
78+ var subs []db.SubData
96 79
97- existing, err := s.articles.GetSubscription(ctx, userDID, rec.FeedURL)80+ for _, r := range records {
98- if err == nil && existing != nil {81+ var rec SubscriptionRecord
99- if !existing.URI.Valid || existing.URI.String == "" {82+ if err := json.Unmarshal(r.Value, &rec); err != nil {
100- return s.articles.UpdateSubscriptionURI(ctx, userDID, rec.FeedURL, uri, cid)83+ continue
101 }84 }
102- return nil85+ if rec.FeedURL == "" {
86+ continue
87+ }
88+ feeds = append(feeds, &db.Feed{FeedURL: rec.FeedURL, Title: db.NullStr(rec.Title)})
89+ subs = append(subs, db.SubData{
90+ FeedURL: rec.FeedURL,
91+ Title: rec.Title,
92+ Category: rec.Category,
93+ URI: r.URI,
94+ CID: r.CID,
95+ })
103 }96 }
104 97
105- err = s.articles.CreateSubscription(ctx, userDID, rec.FeedURL, rec.Title, rec.Category, uri, cid)98+ if len(feeds) > 0 {
106- if errors.Is(err, db.ErrDuplicateSubscription) {99+ _ = s.articles.BatchUpsertFeeds(ctx, feeds)
107- return nil
108 }100 }
109- return err101+ return s.articles.BatchReconcileSubscriptions(ctx, userDID, subs)
110 }102 }
111 103
112-func (s *Sync) reconcileSkyreaderSubscription(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {104+func (s *Sync) batchReconcileSkyreaderSubscriptions(ctx context.Context, userDID string, records []Record) error {
113- var rec SkyreaderSubscriptionRecord105+ var feeds []*db.Feed
114- if err := json.Unmarshal(value, &rec); err != nil {106+ var subs []db.SubData
115- return err
116- }
117-
118- if rec.FeedURL == "" {
119- return nil
120- }
121 107
122- f := &db.Feed{FeedURL: rec.FeedURL, Title: db.NullStr(rec.Title), SiteURL: db.NullStr(rec.SiteURL)}108+ for _, r := range records {
123- _ = s.articles.UpsertFeed(ctx, f)109+ var rec SkyreaderSubscriptionRecord
124-110+ if err := json.Unmarshal(r.Value, &rec); err != nil {
125- existing, err := s.articles.GetSubscription(ctx, userDID, rec.FeedURL)111+ continue
126- if err == nil && existing != nil {
127- if !existing.URI.Valid || existing.URI.String == "" {
128- return s.articles.UpdateSubscriptionURI(ctx, userDID, rec.FeedURL, uri, cid)
129 }112 }
130- return nil113+ if rec.FeedURL == "" {
114+ continue
115+ }
116+ feeds = append(feeds, &db.Feed{FeedURL: rec.FeedURL, Title: db.NullStr(rec.Title), SiteURL: db.NullStr(rec.SiteURL)})
117+ subs = append(subs, db.SubData{
118+ FeedURL: rec.FeedURL,
119+ Title: rec.Title,
120+ URI: r.URI,
121+ CID: r.CID,
122+ })
131 }123 }
132 124
133- err = s.articles.CreateSubscription(ctx, userDID, rec.FeedURL, rec.Title, "", uri, cid)125+ if len(feeds) > 0 {
134- if errors.Is(err, db.ErrDuplicateSubscription) {126+ _ = s.articles.BatchUpsertFeeds(ctx, feeds)
135- return nil
136 }127 }
137- return err128+ return s.articles.BatchReconcileSubscriptions(ctx, userDID, subs)
138 }129 }
139 130
140-func (s *Sync) reconcileLike(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {131+func (s *Sync) batchReconcileLikes(ctx context.Context, userDID string, records []Record) error {
141- var rec LikeRecord132+ var likes []*db.Like
142- if err := json.Unmarshal(value, &rec); err != nil {
143- return err
144- }
145-
146- if rec.FeedURL == "" || rec.ArticleURL == "" {
147- return nil
148- }
149-
150- exists, err := s.articles.HasLiked(ctx, userDID, rec.FeedURL, rec.ArticleURL)
151- if err != nil {
152- return err
153- }
154- if exists {
155- return nil
156- }
157 133
158- t, _ := time.Parse(time.RFC3339, rec.CreatedAt)134+ for _, r := range records {
159- like := &db.Like{135+ var rec LikeRecord
160- URI: uri,136+ if err := json.Unmarshal(r.Value, &rec); err != nil {
161- AuthorDID: userDID,137+ continue
162- FeedURL: rec.FeedURL,138+ }
163- ArticleURL: rec.ArticleURL,139+ if rec.FeedURL == "" || rec.ArticleURL == "" {
164- CreatedAt: db.NullTime(t),140+ continue
165- CID: db.NullStr(cid),141+ }
166- }142+ t, _ := time.Parse(time.RFC3339, rec.CreatedAt)
167- err = s.articles.CreateLike(ctx, like)143+ likes = append(likes, &db.Like{
168- if errors.Is(err, db.ErrDuplicateLike) {144+ URI: r.URI,
169- return nil145+ AuthorDID: userDID,
170- }146+ FeedURL: rec.FeedURL,
171- return err147+ ArticleURL: rec.ArticleURL,
148+ CreatedAt: db.NullTime(t),
149+ CID: db.NullStr(r.CID),
150+ })
151+ }
152+
153+ return s.articles.BatchCreateLikes(ctx, likes)
172 }154 }
173 155
174-func (s *Sync) reconcileAnnotation(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {156+func (s *Sync) batchReconcileAnnotations(ctx context.Context, userDID string, records []Record) error {
175- var rec AnnotationRecord157+ var annotations []*db.Annotation
176- if err := json.Unmarshal(value, &rec); err != nil {
177- return err
178- }
179-
180- if rec.FeedURL == "" || rec.ArticleURL == "" {
181- return nil
182- }
183 158
184- exists, err := s.articles.AnnotationExists(ctx, uri)159+ for _, r := range records {
185- if err != nil || exists {160+ var rec AnnotationRecord
186- return err161+ if err := json.Unmarshal(r.Value, &rec); err != nil {
162+ continue
163+ }
164+ if rec.FeedURL == "" || rec.ArticleURL == "" {
165+ continue
166+ }
167+ t, _ := time.Parse(time.RFC3339, rec.CreatedAt)
168+ a := &db.Annotation{
169+ URI: r.URI,
170+ AuthorDID: userDID,
171+ FeedURL: rec.FeedURL,
172+ ArticleURL: rec.ArticleURL,
173+ Quote: db.NullStr(rec.Quote),
174+ Note: db.NullStr(rec.Note),
175+ Tags: db.NullStrTags(rec.Tags),
176+ CreatedAt: db.NullTime(t),
177+ CID: db.NullStr(r.CID),
178+ }
179+ if rec.Rating > 0 {
180+ a.Rating = db.NullInt(int64(rec.Rating))
181+ }
182+ annotations = append(annotations, a)
187 }183 }
188 184
189- t, _ := time.Parse(time.RFC3339, rec.CreatedAt)185+ return s.articles.BatchCreateAnnotations(ctx, annotations)
190- a := &db.Annotation{
191- URI: uri,
192- AuthorDID: userDID,
193- FeedURL: rec.FeedURL,
194- ArticleURL: rec.ArticleURL,
195- Quote: db.NullStr(rec.Quote),
196- Note: db.NullStr(rec.Note),
197- Tags: db.NullStrTags(rec.Tags),
198- CreatedAt: db.NullTime(t),
199- CID: db.NullStr(cid),
200- }
201- if rec.Rating > 0 {
202- a.Rating = db.NullInt(int64(rec.Rating))
203- }
204- return s.articles.CreateAnnotation(ctx, a)
205 }186 }
206 187
207-func (s *Sync) reconcileMarginNote(ctx context.Context, userDID, uri, cid string, value json.RawMessage) error {188+func (s *Sync) batchReconcileMarginNotes(ctx context.Context, userDID string, records []Record) error {
208- var rec MarginNoteRecord189+ var annotations []*db.Annotation
209- if err := json.Unmarshal(value, &rec); err != nil {
210- return err
211- }
212 190
213- articleURL, quote, note, tags := rec.ToAnnotation()191+ for _, r := range records {
214- if articleURL == "" {192+ var rec MarginNoteRecord
215- return nil193+ if err := json.Unmarshal(r.Value, &rec); err != nil {
216- }194+ continue
217-195+ }
218- exists, err := s.articles.AnnotationExists(ctx, uri)196+ articleURL, quote, note, tags := rec.ToAnnotation()
219- if err != nil || exists {197+ if articleURL == "" {
220- return err198+ continue
221- }199+ }
222 200
223- feedURL := ""201+ feedURL := ""
224- if article, err := s.articles.GetArticleByURL(ctx, articleURL); err == nil {202+ if article, err := s.articles.GetArticleByURL(ctx, articleURL); err == nil {
225- feedURL = article.FeedURL203+ feedURL = article.FeedURL
226- }204+ }
227 205
228- t, _ := time.Parse(time.RFC3339, rec.CreatedAt)206+ t, _ := time.Parse(time.RFC3339, rec.CreatedAt)
229- a := &db.Annotation{207+ annotations = append(annotations, &db.Annotation{
230- URI: uri,208+ URI: r.URI,
231- AuthorDID: userDID,209+ AuthorDID: userDID,
232- FeedURL: feedURL,210+ FeedURL: feedURL,
233- ArticleURL: articleURL,211+ ArticleURL: articleURL,
234- Quote: db.NullStr(quote),212+ Quote: db.NullStr(quote),
235- Note: db.NullStr(note),213+ Note: db.NullStr(note),
236- Tags: db.NullStrTags(tags),214+ Tags: db.NullStrTags(tags),
237- CreatedAt: db.NullTime(t),215+ CreatedAt: db.NullTime(t),
238- CID: db.NullStr(cid),216+ CID: db.NullStr(r.CID),
239- }217+ })
240- return s.articles.CreateAnnotation(ctx, a)218+ }
219+
220+ return s.articles.BatchCreateAnnotations(ctx, annotations)
241 }221 }
242 222
243 func (s *Sync) syncFollows(ctx context.Context, userDID string) error {223 func (s *Sync) syncFollows(ctx context.Context, userDID string) error {
@@ -266,16 +246,6 @@ func (s *Sync) syncFollows(ctx context.Context, userDID string) error {
266 CID: db.NullStr(r.CID),246 CID: db.NullStr(r.CID),
267 FollowedAt: db.NullTime(t),247 FollowedAt: db.NullTime(t),
268 }248 }
269-
270- // auto onboard followers
271- var handle, displayName, avatarURL string
272- if h, dn, avatar, err := FetchProfile(ctx, rec.Subject); err == nil {
273- handle = h
274- displayName = dn
275- avatarURL = avatar
276- }
277-
278- s.users.CreateUser(ctx, rec.Subject, handle, displayName, avatarURL)
279 }249 }
280 250
281 if next == "" || len(records) == 0 {251 if next == "" || len(records) == 0 {
@@ -289,5 +259,42 @@ func (s *Sync) syncFollows(ctx context.Context, userDID string) error {
289 return nil259 return nil
290 }260 }
291 261
262+ type profileResult struct {
263+ did string
264+ handle string
265+ displayName string
266+ avatarURL string
267+ }
268+
269+ var wg sync.WaitGroup
270+ sem := make(chan struct{}, 10)
271+ results := make([]profileResult, 0, len(activeFollows))
272+ var mu sync.Mutex
273+
274+ for targetDID := range activeFollows {
275+ sem <- struct{}{}
276+ wg.Add(1)
277+ go func(did string) {
278+ defer func() { <-sem }()
279+ defer wg.Done()
280+ var handle, displayName, avatarURL string
281+ if h, dn, avatar, err := FetchProfile(ctx, did); err == nil {
282+ handle = h
283+ displayName = dn
284+ avatarURL = avatar
285+ }
286+ mu.Lock()
287+ results = append(results, profileResult{did, handle, displayName, avatarURL})
288+ mu.Unlock()
289+ }(targetDID)
290+ }
291+ wg.Wait()
292+
293+ var users []db.UserData
294+ for _, r := range results {
295+ users = append(users, db.UserData{DID: r.did, Handle: r.handle, DisplayName: r.displayName, AvatarURL: r.avatarURL})
296+ }
297+ _ = s.users.BatchCreateUsers(ctx, users)
298+
292 return s.users.SyncFollows(ctx, userDID, activeFollows)299 return s.users.SyncFollows(ctx, userDID, activeFollows)
293 }300 }
modified internal/cluster/dismiss.go +1 -1
@@ -79,4 +79,4 @@ func (e *Engine) IsFeedDismissed(ctx context.Context, userDID, feedURL string) (
7979 WHERE user_did = ? AND target_type = 'feed' AND target_id = ?
8080 `, userDID, feedURL).Scan(&count)
8181 return count > 0, err
82-}
\ No newline at end of file
82+}
@@ -79,4 +79,4 @@ func (e *Engine) IsFeedDismissed(ctx context.Context, userDID, feedURL string) (
79 WHERE user_did = ? AND target_type = 'feed' AND target_id = ?79 WHERE user_did = ? AND target_type = 'feed' AND target_id = ?
80 `, userDID, feedURL).Scan(&count)80 `, userDID, feedURL).Scan(&count)
81 return count > 0, err81 return count > 0, err
82-}
\ No newline at end of file\ No newline at end of file
82+}
modified internal/cluster/jaccard.go +1 -1
@@ -366,4 +366,4 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
366366
367367 e.logger.Info("user similarity computed")
368368 return tx.Commit()
369-}
\ No newline at end of file
369+}
@@ -366,4 +366,4 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
366 366
367 e.logger.Info("user similarity computed")367 e.logger.Info("user similarity computed")
368 return tx.Commit()368 return tx.Commit()
369-}
\ No newline at end of file\ No newline at end of file
369+}
modified internal/cluster/jaccard_test.go +133 -118
@@ -12,21 +12,32 @@ import (
1212 "log/slog"
1313 )
1414
15-func setupClusterTestDB(t *testing.T) *db.DB {
15+func setupClusterTestDB(t *testing.T) *db.Databases {
1616 t.Helper()
1717 f, err := os.CreateTemp("", "glean-cluster-test-*.db")
1818 assert.NilError(t, err)
1919 assert.NilError(t, f.Close())
2020 path := f.Name()
21- t.Cleanup(func() { _ = os.Remove(path) })
22-
23- database, err := db.Open(path)
21+ t.Cleanup(func() {
22+ _ = os.Remove(path)
23+ _ = os.Remove(path + "_users")
24+ _ = os.Remove(path + "_users-shm")
25+ _ = os.Remove(path + "_users-wal")
26+ _ = os.Remove(path + "_articles")
27+ _ = os.Remove(path + "_articles-shm")
28+ _ = os.Remove(path + "_articles-wal")
29+ _ = os.Remove(path + "_recs")
30+ _ = os.Remove(path + "_recs-shm")
31+ _ = os.Remove(path + "_recs-wal")
32+ })
33+
34+ dbs, err := db.OpenAll(path)
2435 assert.NilError(t, err)
25- t.Cleanup(func() { _ = database.Close() })
26- return database
36+ t.Cleanup(func() { _ = dbs.Close() })
37+ return dbs
2738 }
2839
29-func seedClusterData(t *testing.T, ctx context.Context, database *db.DB) {
40+func seedClusterData(t *testing.T, ctx context.Context, dbs *db.Databases) {
3041 t.Helper()
3142
3243 users := []struct{ did, handle string }{
@@ -35,7 +46,7 @@ func seedClusterData(t *testing.T, ctx context.Context, database *db.DB) {
3546 {"did:test:carol", "carol"},
3647 }
3748 for _, u := range users {
38- _, err := database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, u.did, u.handle)
49+ _, err := dbs.Users.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, u.did, u.handle)
3950 assert.NilError(t, err)
4051 }
4152
@@ -47,7 +58,7 @@ func seedClusterData(t *testing.T, ctx context.Context, database *db.DB) {
4758 {"https://e.com/feed", "Feed E"},
4859 }
4960 for _, f := range feeds {
50- _, err := database.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type, subscriber_count) VALUES (?, ?, ?, '', 'rss', 2)`, f.url, f.title, f.url)
61+ _, err := dbs.Articles.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type, subscriber_count) VALUES (?, ?, ?, '', 'rss', 2)`, f.url, f.title, f.url)
5162 assert.NilError(t, err)
5263 }
5364
@@ -63,59 +74,63 @@ func seedClusterData(t *testing.T, ctx context.Context, database *db.DB) {
6374 {"did:test:carol", "https://c.com/feed"},
6475 }
6576 for _, s := range subs {
66- _, err := database.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, s.user, s.feed)
77+ _, err := dbs.Articles.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, s.user, s.feed)
6778 assert.NilError(t, err)
6879 }
6980 }
7081
71-func seedFollowData(t *testing.T, ctx context.Context, database *db.DB) {
82+func seedFollowData(t *testing.T, ctx context.Context, dbs *db.Databases) {
7283 t.Helper()
7384 follows := []struct{ user, target string }{
7485 {"did:test:alice", "did:test:bob"},
7586 {"did:test:bob", "did:test:carol"},
7687 }
7788 for _, f := range follows {
78- _, err := database.ExecContext(ctx, `INSERT OR IGNORE INTO follows (user_did, target_did) VALUES (?, ?)`, f.user, f.target)
89+ _, err := dbs.Users.ExecContext(ctx, `INSERT OR IGNORE INTO follows (user_did, target_did) VALUES (?, ?)`, f.user, f.target)
7990 assert.NilError(t, err)
8091 }
8192 }
8293
94+func newTestEngine(dbs *db.Databases) *Engine {
95+ return NewEngine(dbs.Users.DB, slog.Default())
96+}
97+
8398 func TestComputeFeedSimilarity(t *testing.T) {
8499 ctx := context.Background()
85- database := setupClusterTestDB(t)
86- seedClusterData(t, ctx, database)
100+ dbs := setupClusterTestDB(t)
101+ seedClusterData(t, ctx, dbs)
87102
88- engine := NewEngine(database.DB, slog.Default())
103+ engine := newTestEngine(dbs)
89104 err := engine.ComputeFeedSimilarity(ctx)
90105 assert.NilError(t, err)
91106
92107 var count int
93- err = database.QueryRowContext(ctx, `SELECT COUNT(*) FROM feed_similarity`).Scan(&count)
108+ err = dbs.Users.QueryRowContext(ctx, `SELECT COUNT(*) FROM recs.feed_similarity`).Scan(&count)
94109 assert.NilError(t, err)
95110 assert.Assert(t, count > 0, "expected feed similarity pairs")
96111 }
97112
98113 func TestComputeUserSimilarity(t *testing.T) {
99114 ctx := context.Background()
100- database := setupClusterTestDB(t)
101- seedClusterData(t, ctx, database)
115+ dbs := setupClusterTestDB(t)
116+ seedClusterData(t, ctx, dbs)
102117
103- engine := NewEngine(database.DB, slog.Default())
118+ engine := newTestEngine(dbs)
104119 err := engine.ComputeUserSimilarity(ctx)
105120 assert.NilError(t, err)
106121
107122 var count int
108- err = database.QueryRowContext(ctx, `SELECT COUNT(*) FROM user_similarity`).Scan(&count)
123+ err = dbs.Users.QueryRowContext(ctx, `SELECT COUNT(*) FROM recs.user_similarity`).Scan(&count)
109124 assert.NilError(t, err)
110125 assert.Assert(t, count > 0, "expected user similarity pairs")
111126 }
112127
113128 func TestOnDemandFeedRecommendations(t *testing.T) {
114129 ctx := context.Background()
115- database := setupClusterTestDB(t)
116- seedClusterData(t, ctx, database)
130+ dbs := setupClusterTestDB(t)
131+ seedClusterData(t, ctx, dbs)
117132
118- engine := NewEngine(database.DB, slog.Default())
133+ engine := newTestEngine(dbs)
119134 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))
120135 assert.NilError(t, engine.ComputeUserSimilarity(ctx))
121136
@@ -134,10 +149,10 @@ func TestOnDemandFeedRecommendations(t *testing.T) {
134149
135150 func TestNoSelfRecommendations(t *testing.T) {
136151 ctx := context.Background()
137- database := setupClusterTestDB(t)
138- seedClusterData(t, ctx, database)
152+ dbs := setupClusterTestDB(t)
153+ seedClusterData(t, ctx, dbs)
139154
140- engine := NewEngine(database.DB, slog.Default())
155+ engine := newTestEngine(dbs)
141156 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))
142157 assert.NilError(t, engine.ComputeUserSimilarity(ctx))
143158
@@ -157,10 +172,10 @@ func TestNoSelfRecommendations(t *testing.T) {
157172
158173 func TestDismissedFeedsExcluded(t *testing.T) {
159174 ctx := context.Background()
160- database := setupClusterTestDB(t)
161- seedClusterData(t, ctx, database)
175+ dbs := setupClusterTestDB(t)
176+ seedClusterData(t, ctx, dbs)
162177
163- engine := NewEngine(database.DB, slog.Default())
178+ engine := newTestEngine(dbs)
164179 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))
165180 assert.NilError(t, engine.ComputeUserSimilarity(ctx))
166181
@@ -177,10 +192,10 @@ func TestDismissedFeedsExcluded(t *testing.T) {
177192
178193 func TestIsFeedDismissed(t *testing.T) {
179194 ctx := context.Background()
180- database := setupClusterTestDB(t)
181- seedClusterData(t, ctx, database)
195+ dbs := setupClusterTestDB(t)
196+ seedClusterData(t, ctx, dbs)
182197
183- engine := NewEngine(database.DB, slog.Default())
198+ engine := newTestEngine(dbs)
184199
185200 dismissed, err := engine.IsFeedDismissed(ctx, "did:test:alice", "https://a.com/feed")
186201 assert.NilError(t, err)
@@ -195,10 +210,10 @@ func TestIsFeedDismissed(t *testing.T) {
195210
196211 func TestRecordImpressions(t *testing.T) {
197212 ctx := context.Background()
198- database := setupClusterTestDB(t)
199- seedClusterData(t, ctx, database)
213+ dbs := setupClusterTestDB(t)
214+ seedClusterData(t, ctx, dbs)
200215
201- engine := NewEngine(database.DB, slog.Default())
216+ engine := newTestEngine(dbs)
202217
203218 impressions := []Impression{
204219 {TargetType: "feed", TargetID: "https://a.com/feed"},
@@ -207,24 +222,24 @@ func TestRecordImpressions(t *testing.T) {
207222 assert.NilError(t, engine.RecordImpressions(ctx, "did:test:alice", impressions))
208223
209224 var count int
210- assert.NilError(t, database.QueryRowContext(ctx,
211- `SELECT COUNT(*) FROM recommendation_impressions WHERE user_did = 'did:test:alice'`).Scan(&count))
225+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
226+ `SELECT COUNT(*) FROM recs.recommendation_impressions WHERE user_did = 'did:test:alice'`).Scan(&count))
212227 assert.Equal(t, count, 2)
213228
214229 assert.NilError(t, engine.RecordImpressions(ctx, "did:test:alice", impressions))
215230
216231 var shownCount int
217- assert.NilError(t, database.QueryRowContext(ctx,
218- `SELECT shown_count FROM recommendation_impressions WHERE user_did = 'did:test:alice' AND target_id = 'https://a.com/feed'`).Scan(&shownCount))
232+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
233+ `SELECT shown_count FROM recs.recommendation_impressions WHERE user_did = 'did:test:alice' AND target_id = 'https://a.com/feed'`).Scan(&shownCount))
219234 assert.Equal(t, shownCount, 2, "shown_count should increment on repeated impression")
220235 }
221236
222237 func TestMarkImpressionActed(t *testing.T) {
223238 ctx := context.Background()
224- database := setupClusterTestDB(t)
225- seedClusterData(t, ctx, database)
239+ dbs := setupClusterTestDB(t)
240+ seedClusterData(t, ctx, dbs)
226241
227- engine := NewEngine(database.DB, slog.Default())
242+ engine := newTestEngine(dbs)
228243
229244 impressions := []Impression{{TargetType: "feed", TargetID: "https://a.com/feed"}}
230245 assert.NilError(t, engine.RecordImpressions(ctx, "did:test:alice", impressions))
@@ -232,43 +247,43 @@ func TestMarkImpressionActed(t *testing.T) {
232247 assert.NilError(t, engine.MarkImpressionActed(ctx, "did:test:alice", "feed", "https://a.com/feed"))
233248
234249 var acted bool
235- assert.NilError(t, database.QueryRowContext(ctx,
236- `SELECT acted FROM recommendation_impressions WHERE user_did = 'did:test:alice' AND target_id = 'https://a.com/feed'`).Scan(&acted))
250+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
251+ `SELECT acted FROM recs.recommendation_impressions WHERE user_did = 'did:test:alice' AND target_id = 'https://a.com/feed'`).Scan(&acted))
237252 assert.Assert(t, acted, "impression should be marked as acted")
238253 }
239254
240255 func TestComputeFollowDistances(t *testing.T) {
241256 ctx := context.Background()
242- database := setupClusterTestDB(t)
243- seedClusterData(t, ctx, database)
244- seedFollowData(t, ctx, database)
257+ dbs := setupClusterTestDB(t)
258+ seedClusterData(t, ctx, dbs)
259+ seedFollowData(t, ctx, dbs)
245260
246- engine := NewEngine(database.DB, slog.Default())
261+ engine := newTestEngine(dbs)
247262 assert.NilError(t, engine.ComputeFollowDistances(ctx))
248263
249264 var d1, d2 int
250- assert.NilError(t, database.QueryRowContext(ctx,
251- `SELECT COUNT(*) FROM follow_distances WHERE distance = 1`).Scan(&d1))
252- assert.NilError(t, database.QueryRowContext(ctx,
253- `SELECT COUNT(*) FROM follow_distances WHERE distance = 2`).Scan(&d2))
265+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
266+ `SELECT COUNT(*) FROM recs.follow_distances WHERE distance = 1`).Scan(&d1))
267+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
268+ `SELECT COUNT(*) FROM recs.follow_distances WHERE distance = 2`).Scan(&d2))
254269 assert.Assert(t, d1 >= 2, "expected at least 2 direct follow distances")
255270 assert.Assert(t, d2 >= 1, "expected at least 1 two-hop distance (alice -> bob -> carol)")
256271
257272 var dist int
258- assert.NilError(t, database.QueryRowContext(ctx,
259- `SELECT distance FROM follow_distances WHERE user_a = 'did:test:alice' AND user_b = 'did:test:carol'`).Scan(&dist))
273+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
274+ `SELECT distance FROM recs.follow_distances WHERE user_a = 'did:test:alice' AND user_b = 'did:test:carol'`).Scan(&dist))
260275 assert.Equal(t, dist, 2, "alice should be 2 hops from carol")
261276 }
262277
263278 func TestAutoDismissStale(t *testing.T) {
264279 ctx := context.Background()
265- database := setupClusterTestDB(t)
266- seedClusterData(t, ctx, database)
280+ dbs := setupClusterTestDB(t)
281+ seedClusterData(t, ctx, dbs)
267282
268- engine := NewEngine(database.DB, slog.Default())
283+ engine := newTestEngine(dbs)
269284
270- _, err := database.ExecContext(ctx, `
271- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
285+ _, err := dbs.Users.ExecContext(ctx, `
286+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
272287 VALUES ('did:test:alice', 'feed', 'https://stale.com/feed', datetime('now', '-31 days'), datetime('now'), 20, 0)
273288 `)
274289 assert.NilError(t, err)
@@ -282,13 +297,13 @@ func TestAutoDismissStale(t *testing.T) {
282297
283298 func TestAutoDismissStale_DoesNotDismissRecent(t *testing.T) {
284299 ctx := context.Background()
285- database := setupClusterTestDB(t)
286- seedClusterData(t, ctx, database)
300+ dbs := setupClusterTestDB(t)
301+ seedClusterData(t, ctx, dbs)
287302
288- engine := NewEngine(database.DB, slog.Default())
303+ engine := newTestEngine(dbs)
289304
290- _, err := database.ExecContext(ctx, `
291- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
305+ _, err := dbs.Users.ExecContext(ctx, `
306+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
292307 VALUES ('did:test:alice', 'feed', 'https://recent.com/feed', datetime('now'), datetime('now'), 5, 0)
293308 `)
294309 assert.NilError(t, err)
@@ -302,13 +317,13 @@ func TestAutoDismissStale_DoesNotDismissRecent(t *testing.T) {
302317
303318 func TestAutoDismissStale_DoesNotDismissActed(t *testing.T) {
304319 ctx := context.Background()
305- database := setupClusterTestDB(t)
306- seedClusterData(t, ctx, database)
320+ dbs := setupClusterTestDB(t)
321+ seedClusterData(t, ctx, dbs)
307322
308- engine := NewEngine(database.DB, slog.Default())
323+ engine := newTestEngine(dbs)
309324
310- _, err := database.ExecContext(ctx, `
311- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
325+ _, err := dbs.Users.ExecContext(ctx, `
326+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
312327 VALUES ('did:test:alice', 'feed', 'https://acted.com/feed', datetime('now', '-31 days'), datetime('now'), 20, 1)
313328 `)
314329 assert.NilError(t, err)
@@ -361,10 +376,10 @@ func TestDiversityFiltering_EmptySiteURL(t *testing.T) {
361376
362377 func TestSignalWeights_Default(t *testing.T) {
363378 ctx := context.Background()
364- database := setupClusterTestDB(t)
365- seedClusterData(t, ctx, database)
379+ dbs := setupClusterTestDB(t)
380+ seedClusterData(t, ctx, dbs)
366381
367- engine := NewEngine(database.DB, slog.Default())
382+ engine := newTestEngine(dbs)
368383 w := engine.GetWeights(ctx, "did:test:alice")
369384
370385 assert.Equal(t, w.WSub, 1.0)
@@ -377,19 +392,19 @@ func TestSignalWeights_Default(t *testing.T) {
377392
378393 func TestSignalWeights_RewardPenalize(t *testing.T) {
379394 ctx := context.Background()
380- database := setupClusterTestDB(t)
381- seedClusterData(t, ctx, database)
395+ dbs := setupClusterTestDB(t)
396+ seedClusterData(t, ctx, dbs)
382397
383- engine := NewEngine(database.DB, slog.Default())
398+ engine := newTestEngine(dbs)
384399
385- _, err := database.ExecContext(ctx, `
386- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
400+ _, err := dbs.Users.ExecContext(ctx, `
401+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
387402 VALUES ('did:test:alice', 'feed', 'https://a.com/feed', datetime('now'), datetime('now'), 1, 1)
388403 `)
389404 assert.NilError(t, err)
390405 for i := range minActionsTune {
391- _, err = database.ExecContext(ctx, `
392- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
406+ _, err = dbs.Users.ExecContext(ctx, `
407+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
393408 VALUES ('did:test:alice', 'feed', ?, datetime('now'), datetime('now'), 1, 1)
394409 `, fmt.Sprintf("https://%d.com/feed", i))
395410 assert.NilError(t, err)
@@ -403,19 +418,19 @@ func TestSignalWeights_RewardPenalize(t *testing.T) {
403418
404419 func TestColdStartRecommendations(t *testing.T) {
405420 ctx := context.Background()
406- database := setupClusterTestDB(t)
407- seedClusterData(t, ctx, database)
408- seedFollowData(t, ctx, database)
421+ dbs := setupClusterTestDB(t)
422+ seedClusterData(t, ctx, dbs)
423+ seedFollowData(t, ctx, dbs)
409424
410- engine := NewEngine(database.DB, slog.Default())
425+ engine := newTestEngine(dbs)
411426 assert.NilError(t, engine.ComputeFollowDistances(ctx))
412427
413- _, err := database.ExecContext(ctx, `UPDATE feeds SET subscriber_count = 2 WHERE feed_url = 'https://a.com/feed'`)
428+ _, err := dbs.Articles.ExecContext(ctx, `UPDATE feeds SET subscriber_count = 2 WHERE feed_url = 'https://a.com/feed'`)
414429 assert.NilError(t, err)
415- _, err = database.ExecContext(ctx, `UPDATE feeds SET subscriber_count = 2 WHERE feed_url = 'https://b.com/feed'`)
430+ _, err = dbs.Articles.ExecContext(ctx, `UPDATE feeds SET subscriber_count = 2 WHERE feed_url = 'https://b.com/feed'`)
416431 assert.NilError(t, err)
417432
418- _, err = database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:newuser", "newuser")
433+ _, err = dbs.Users.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:newuser", "newuser")
419434 assert.NilError(t, err)
420435
421436 recs, err := engine.ColdStartRecommendations(ctx, "did:test:newuser", 10)
@@ -425,11 +440,11 @@ func TestColdStartRecommendations(t *testing.T) {
425440
426441 func TestColdStartRecommendations_NotTriggeredForEstablishedUser(t *testing.T) {
427442 ctx := context.Background()
428- database := setupClusterTestDB(t)
429- seedClusterData(t, ctx, database)
430- seedFollowData(t, ctx, database)
443+ dbs := setupClusterTestDB(t)
444+ seedClusterData(t, ctx, dbs)
445+ seedFollowData(t, ctx, dbs)
431446
432- engine := NewEngine(database.DB, slog.Default())
447+ engine := newTestEngine(dbs)
433448 assert.NilError(t, engine.ComputeFollowDistances(ctx))
434449
435450 recs, err := engine.ColdStartRecommendations(ctx, "did:test:alice", 10)
@@ -439,10 +454,10 @@ func TestColdStartRecommendations_NotTriggeredForEstablishedUser(t *testing.T) {
439454
440455 func TestOnDemandPeopleRecommendations(t *testing.T) {
441456 ctx := context.Background()
442- database := setupClusterTestDB(t)
443- seedClusterData(t, ctx, database)
457+ dbs := setupClusterTestDB(t)
458+ seedClusterData(t, ctx, dbs)
444459
445- engine := NewEngine(database.DB, slog.Default())
460+ engine := newTestEngine(dbs)
446461 assert.NilError(t, engine.ComputeUserSimilarity(ctx))
447462
448463 recs, err := engine.GetPeopleRecommendations(ctx, "did:test:carol", 10)
@@ -452,80 +467,80 @@ func TestOnDemandPeopleRecommendations(t *testing.T) {
452467
453468 func TestDismissArticle(t *testing.T) {
454469 ctx := context.Background()
455- database := setupClusterTestDB(t)
456- seedClusterData(t, ctx, database)
470+ dbs := setupClusterTestDB(t)
471+ seedClusterData(t, ctx, dbs)
457472
458- engine := NewEngine(database.DB, slog.Default())
473+ engine := newTestEngine(dbs)
459474
460475 assert.NilError(t, engine.DismissArticle(ctx, "did:test:alice", "https://a.com/article1", "not_interested"))
461476
462477 var count int
463- assert.NilError(t, database.QueryRowContext(ctx,
464- `SELECT COUNT(*) FROM dismissed_recommendations WHERE user_did = 'did:test:alice' AND target_type = 'article'`).Scan(&count))
478+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
479+ `SELECT COUNT(*) FROM recs.dismissed_recommendations WHERE user_did = 'did:test:alice' AND target_type = 'article'`).Scan(&count))
465480 assert.Equal(t, count, 1)
466481 }
467482
468483 func TestComputeSignalProfiles(t *testing.T) {
469484 ctx := context.Background()
470- database := setupClusterTestDB(t)
471- seedClusterData(t, ctx, database)
485+ dbs := setupClusterTestDB(t)
486+ seedClusterData(t, ctx, dbs)
472487
473- engine := NewEngine(database.DB, slog.Default())
488+ engine := newTestEngine(dbs)
474489 assert.NilError(t, engine.ComputeSignalProfiles(ctx))
475490
476491 var count int
477- assert.NilError(t, database.QueryRowContext(ctx, `SELECT COUNT(*) FROM user_signal_profiles`).Scan(&count))
492+ assert.NilError(t, dbs.Users.QueryRowContext(ctx, `SELECT COUNT(*) FROM recs.user_signal_profiles`).Scan(&count))
478493 assert.Assert(t, count >= 3, "expected signal profiles for all users")
479494 }
480495
481496 func TestDismissFeed_Idempotent(t *testing.T) {
482497 ctx := context.Background()
483- database := setupClusterTestDB(t)
484- seedClusterData(t, ctx, database)
498+ dbs := setupClusterTestDB(t)
499+ seedClusterData(t, ctx, dbs)
485500
486- engine := NewEngine(database.DB, slog.Default())
501+ engine := newTestEngine(dbs)
487502
488503 assert.NilError(t, engine.DismissFeed(ctx, "did:test:alice", "https://a.com/feed", "reason1"))
489504 assert.NilError(t, engine.DismissFeed(ctx, "did:test:alice", "https://a.com/feed", "reason2"))
490505
491506 var count int
492- assert.NilError(t, database.QueryRowContext(ctx,
493- `SELECT COUNT(*) FROM dismissed_recommendations WHERE user_did = 'did:test:alice' AND target_type = 'feed'`).Scan(&count))
507+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
508+ `SELECT COUNT(*) FROM recs.dismissed_recommendations WHERE user_did = 'did:test:alice' AND target_type = 'feed'`).Scan(&count))
494509 assert.Equal(t, count, 1, "duplicate dismiss should not create extra rows")
495510 }
496511
497512 func TestDescriptionBasedFeedSimilarity(t *testing.T) {
498513 ctx := context.Background()
499- database := setupClusterTestDB(t)
514+ dbs := setupClusterTestDB(t)
500515
501- _, err := database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:alice", "alice")
516+ _, err := dbs.Users.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:alice", "alice")
502517 assert.NilError(t, err)
503- _, err = database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:bob", "bob")
518+ _, err = dbs.Users.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:bob", "bob")
504519 assert.NilError(t, err)
505520
506- _, err = database.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type) VALUES (?, ?, ?, ?, 'rss')`,
521+ _, err = dbs.Articles.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type) VALUES (?, ?, ?, ?, 'rss')`,
507522 "https://go.com/feed", "Go Blog", "https://go.com", "programming language golang software development")
508523 assert.NilError(t, err)
509- _, err = database.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type) VALUES (?, ?, ?, ?, 'rss')`,
524+ _, err = dbs.Articles.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type) VALUES (?, ?, ?, ?, 'rss')`,
510525 "https://rust.com/feed", "Rust Blog", "https://rust.com", "programming language rust software development")
511526 assert.NilError(t, err)
512527
513- _, err = database.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, "did:test:alice", "https://go.com/feed")
528+ _, err = dbs.Articles.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, "did:test:alice", "https://go.com/feed")
514529 assert.NilError(t, err)
515- _, err = database.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, "did:test:bob", "https://rust.com/feed")
530+ _, err = dbs.Articles.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, "did:test:bob", "https://rust.com/feed")
516531 assert.NilError(t, err)
517532
518- engine := NewEngine(database.DB, slog.Default())
533+ engine := newTestEngine(dbs)
519534 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))
520535
521536 var count int
522- assert.NilError(t, database.QueryRowContext(ctx, `SELECT COUNT(*) FROM feed_similarity`).Scan(&count))
537+ assert.NilError(t, dbs.Users.QueryRowContext(ctx, `SELECT COUNT(*) FROM recs.feed_similarity`).Scan(&count))
523538 assert.Assert(t, count >= 0, "description-based similarity should produce pairs")
524539
525540 if count > 0 {
526541 var jaccard float64
527- assert.NilError(t, database.QueryRowContext(ctx,
528- `SELECT jaccard FROM feed_similarity WHERE feed_a = ? AND feed_b = ?`,
542+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
543+ `SELECT jaccard FROM recs.feed_similarity WHERE feed_a = ? AND feed_b = ?`,
529544 "https://go.com/feed", "https://rust.com/feed").Scan(&jaccard))
530545 assert.Assert(t, jaccard > 0, "description word overlap should boost similarity")
531546 }
@@ -12,21 +12,32 @@ import (
12 "log/slog"12 "log/slog"
13 )13 )
14 14
15-func setupClusterTestDB(t *testing.T) *db.DB {15+func setupClusterTestDB(t *testing.T) *db.Databases {
16 t.Helper()16 t.Helper()
17 f, err := os.CreateTemp("", "glean-cluster-test-*.db")17 f, err := os.CreateTemp("", "glean-cluster-test-*.db")
18 assert.NilError(t, err)18 assert.NilError(t, err)
19 assert.NilError(t, f.Close())19 assert.NilError(t, f.Close())
20 path := f.Name()20 path := f.Name()
21- t.Cleanup(func() { _ = os.Remove(path) })21+ t.Cleanup(func() {
22-22+ _ = os.Remove(path)
23- database, err := db.Open(path)23+ _ = os.Remove(path + "_users")
24+ _ = os.Remove(path + "_users-shm")
25+ _ = os.Remove(path + "_users-wal")
26+ _ = os.Remove(path + "_articles")
27+ _ = os.Remove(path + "_articles-shm")
28+ _ = os.Remove(path + "_articles-wal")
29+ _ = os.Remove(path + "_recs")
30+ _ = os.Remove(path + "_recs-shm")
31+ _ = os.Remove(path + "_recs-wal")
32+ })
33+
34+ dbs, err := db.OpenAll(path)
24 assert.NilError(t, err)35 assert.NilError(t, err)
25- t.Cleanup(func() { _ = database.Close() })36+ t.Cleanup(func() { _ = dbs.Close() })
26- return database37+ return dbs
27 }38 }
28 39
29-func seedClusterData(t *testing.T, ctx context.Context, database *db.DB) {40+func seedClusterData(t *testing.T, ctx context.Context, dbs *db.Databases) {
30 t.Helper()41 t.Helper()
31 42
32 users := []struct{ did, handle string }{43 users := []struct{ did, handle string }{
@@ -35,7 +46,7 @@ func seedClusterData(t *testing.T, ctx context.Context, database *db.DB) {
35 {"did:test:carol", "carol"},46 {"did:test:carol", "carol"},
36 }47 }
37 for _, u := range users {48 for _, u := range users {
38- _, err := database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, u.did, u.handle)49+ _, err := dbs.Users.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, u.did, u.handle)
39 assert.NilError(t, err)50 assert.NilError(t, err)
40 }51 }
41 52
@@ -47,7 +58,7 @@ func seedClusterData(t *testing.T, ctx context.Context, database *db.DB) {
47 {"https://e.com/feed", "Feed E"},58 {"https://e.com/feed", "Feed E"},
48 }59 }
49 for _, f := range feeds {60 for _, f := range feeds {
50- _, err := database.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type, subscriber_count) VALUES (?, ?, ?, '', 'rss', 2)`, f.url, f.title, f.url)61+ _, err := dbs.Articles.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type, subscriber_count) VALUES (?, ?, ?, '', 'rss', 2)`, f.url, f.title, f.url)
51 assert.NilError(t, err)62 assert.NilError(t, err)
52 }63 }
53 64
@@ -63,59 +74,63 @@ func seedClusterData(t *testing.T, ctx context.Context, database *db.DB) {
63 {"did:test:carol", "https://c.com/feed"},74 {"did:test:carol", "https://c.com/feed"},
64 }75 }
65 for _, s := range subs {76 for _, s := range subs {
66- _, err := database.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, s.user, s.feed)77+ _, err := dbs.Articles.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, s.user, s.feed)
67 assert.NilError(t, err)78 assert.NilError(t, err)
68 }79 }
69 }80 }
70 81
71-func seedFollowData(t *testing.T, ctx context.Context, database *db.DB) {82+func seedFollowData(t *testing.T, ctx context.Context, dbs *db.Databases) {
72 t.Helper()83 t.Helper()
73 follows := []struct{ user, target string }{84 follows := []struct{ user, target string }{
74 {"did:test:alice", "did:test:bob"},85 {"did:test:alice", "did:test:bob"},
75 {"did:test:bob", "did:test:carol"},86 {"did:test:bob", "did:test:carol"},
76 }87 }
77 for _, f := range follows {88 for _, f := range follows {
78- _, err := database.ExecContext(ctx, `INSERT OR IGNORE INTO follows (user_did, target_did) VALUES (?, ?)`, f.user, f.target)89+ _, err := dbs.Users.ExecContext(ctx, `INSERT OR IGNORE INTO follows (user_did, target_did) VALUES (?, ?)`, f.user, f.target)
79 assert.NilError(t, err)90 assert.NilError(t, err)
80 }91 }
81 }92 }
82 93
94+func newTestEngine(dbs *db.Databases) *Engine {
95+ return NewEngine(dbs.Users.DB, slog.Default())
96+}
97+
83 func TestComputeFeedSimilarity(t *testing.T) {98 func TestComputeFeedSimilarity(t *testing.T) {
84 ctx := context.Background()99 ctx := context.Background()
85- database := setupClusterTestDB(t)100+ dbs := setupClusterTestDB(t)
86- seedClusterData(t, ctx, database)101+ seedClusterData(t, ctx, dbs)
87 102
88- engine := NewEngine(database.DB, slog.Default())103+ engine := newTestEngine(dbs)
89 err := engine.ComputeFeedSimilarity(ctx)104 err := engine.ComputeFeedSimilarity(ctx)
90 assert.NilError(t, err)105 assert.NilError(t, err)
91 106
92 var count int107 var count int
93- err = database.QueryRowContext(ctx, `SELECT COUNT(*) FROM feed_similarity`).Scan(&count)108+ err = dbs.Users.QueryRowContext(ctx, `SELECT COUNT(*) FROM recs.feed_similarity`).Scan(&count)
94 assert.NilError(t, err)109 assert.NilError(t, err)
95 assert.Assert(t, count > 0, "expected feed similarity pairs")110 assert.Assert(t, count > 0, "expected feed similarity pairs")
96 }111 }
97 112
98 func TestComputeUserSimilarity(t *testing.T) {113 func TestComputeUserSimilarity(t *testing.T) {
99 ctx := context.Background()114 ctx := context.Background()
100- database := setupClusterTestDB(t)115+ dbs := setupClusterTestDB(t)
101- seedClusterData(t, ctx, database)116+ seedClusterData(t, ctx, dbs)
102 117
103- engine := NewEngine(database.DB, slog.Default())118+ engine := newTestEngine(dbs)
104 err := engine.ComputeUserSimilarity(ctx)119 err := engine.ComputeUserSimilarity(ctx)
105 assert.NilError(t, err)120 assert.NilError(t, err)
106 121
107 var count int122 var count int
108- err = database.QueryRowContext(ctx, `SELECT COUNT(*) FROM user_similarity`).Scan(&count)123+ err = dbs.Users.QueryRowContext(ctx, `SELECT COUNT(*) FROM recs.user_similarity`).Scan(&count)
109 assert.NilError(t, err)124 assert.NilError(t, err)
110 assert.Assert(t, count > 0, "expected user similarity pairs")125 assert.Assert(t, count > 0, "expected user similarity pairs")
111 }126 }
112 127
113 func TestOnDemandFeedRecommendations(t *testing.T) {128 func TestOnDemandFeedRecommendations(t *testing.T) {
114 ctx := context.Background()129 ctx := context.Background()
115- database := setupClusterTestDB(t)130+ dbs := setupClusterTestDB(t)
116- seedClusterData(t, ctx, database)131+ seedClusterData(t, ctx, dbs)
117 132
118- engine := NewEngine(database.DB, slog.Default())133+ engine := newTestEngine(dbs)
119 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))134 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))
120 assert.NilError(t, engine.ComputeUserSimilarity(ctx))135 assert.NilError(t, engine.ComputeUserSimilarity(ctx))
121 136
@@ -134,10 +149,10 @@ func TestOnDemandFeedRecommendations(t *testing.T) {
134 149
135 func TestNoSelfRecommendations(t *testing.T) {150 func TestNoSelfRecommendations(t *testing.T) {
136 ctx := context.Background()151 ctx := context.Background()
137- database := setupClusterTestDB(t)152+ dbs := setupClusterTestDB(t)
138- seedClusterData(t, ctx, database)153+ seedClusterData(t, ctx, dbs)
139 154
140- engine := NewEngine(database.DB, slog.Default())155+ engine := newTestEngine(dbs)
141 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))156 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))
142 assert.NilError(t, engine.ComputeUserSimilarity(ctx))157 assert.NilError(t, engine.ComputeUserSimilarity(ctx))
143 158
@@ -157,10 +172,10 @@ func TestNoSelfRecommendations(t *testing.T) {
157 172
158 func TestDismissedFeedsExcluded(t *testing.T) {173 func TestDismissedFeedsExcluded(t *testing.T) {
159 ctx := context.Background()174 ctx := context.Background()
160- database := setupClusterTestDB(t)175+ dbs := setupClusterTestDB(t)
161- seedClusterData(t, ctx, database)176+ seedClusterData(t, ctx, dbs)
162 177
163- engine := NewEngine(database.DB, slog.Default())178+ engine := newTestEngine(dbs)
164 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))179 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))
165 assert.NilError(t, engine.ComputeUserSimilarity(ctx))180 assert.NilError(t, engine.ComputeUserSimilarity(ctx))
166 181
@@ -177,10 +192,10 @@ func TestDismissedFeedsExcluded(t *testing.T) {
177 192
178 func TestIsFeedDismissed(t *testing.T) {193 func TestIsFeedDismissed(t *testing.T) {
179 ctx := context.Background()194 ctx := context.Background()
180- database := setupClusterTestDB(t)195+ dbs := setupClusterTestDB(t)
181- seedClusterData(t, ctx, database)196+ seedClusterData(t, ctx, dbs)
182 197
183- engine := NewEngine(database.DB, slog.Default())198+ engine := newTestEngine(dbs)
184 199
185 dismissed, err := engine.IsFeedDismissed(ctx, "did:test:alice", "https://a.com/feed")200 dismissed, err := engine.IsFeedDismissed(ctx, "did:test:alice", "https://a.com/feed")
186 assert.NilError(t, err)201 assert.NilError(t, err)
@@ -195,10 +210,10 @@ func TestIsFeedDismissed(t *testing.T) {
195 210
196 func TestRecordImpressions(t *testing.T) {211 func TestRecordImpressions(t *testing.T) {
197 ctx := context.Background()212 ctx := context.Background()
198- database := setupClusterTestDB(t)213+ dbs := setupClusterTestDB(t)
199- seedClusterData(t, ctx, database)214+ seedClusterData(t, ctx, dbs)
200 215
201- engine := NewEngine(database.DB, slog.Default())216+ engine := newTestEngine(dbs)
202 217
203 impressions := []Impression{218 impressions := []Impression{
204 {TargetType: "feed", TargetID: "https://a.com/feed"},219 {TargetType: "feed", TargetID: "https://a.com/feed"},
@@ -207,24 +222,24 @@ func TestRecordImpressions(t *testing.T) {
207 assert.NilError(t, engine.RecordImpressions(ctx, "did:test:alice", impressions))222 assert.NilError(t, engine.RecordImpressions(ctx, "did:test:alice", impressions))
208 223
209 var count int224 var count int
210- assert.NilError(t, database.QueryRowContext(ctx,225+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
211- `SELECT COUNT(*) FROM recommendation_impressions WHERE user_did = 'did:test:alice'`).Scan(&count))226+ `SELECT COUNT(*) FROM recs.recommendation_impressions WHERE user_did = 'did:test:alice'`).Scan(&count))
212 assert.Equal(t, count, 2)227 assert.Equal(t, count, 2)
213 228
214 assert.NilError(t, engine.RecordImpressions(ctx, "did:test:alice", impressions))229 assert.NilError(t, engine.RecordImpressions(ctx, "did:test:alice", impressions))
215 230
216 var shownCount int231 var shownCount int
217- assert.NilError(t, database.QueryRowContext(ctx,232+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
218- `SELECT shown_count FROM recommendation_impressions WHERE user_did = 'did:test:alice' AND target_id = 'https://a.com/feed'`).Scan(&shownCount))233+ `SELECT shown_count FROM recs.recommendation_impressions WHERE user_did = 'did:test:alice' AND target_id = 'https://a.com/feed'`).Scan(&shownCount))
219 assert.Equal(t, shownCount, 2, "shown_count should increment on repeated impression")234 assert.Equal(t, shownCount, 2, "shown_count should increment on repeated impression")
220 }235 }
221 236
222 func TestMarkImpressionActed(t *testing.T) {237 func TestMarkImpressionActed(t *testing.T) {
223 ctx := context.Background()238 ctx := context.Background()
224- database := setupClusterTestDB(t)239+ dbs := setupClusterTestDB(t)
225- seedClusterData(t, ctx, database)240+ seedClusterData(t, ctx, dbs)
226 241
227- engine := NewEngine(database.DB, slog.Default())242+ engine := newTestEngine(dbs)
228 243
229 impressions := []Impression{{TargetType: "feed", TargetID: "https://a.com/feed"}}244 impressions := []Impression{{TargetType: "feed", TargetID: "https://a.com/feed"}}
230 assert.NilError(t, engine.RecordImpressions(ctx, "did:test:alice", impressions))245 assert.NilError(t, engine.RecordImpressions(ctx, "did:test:alice", impressions))
@@ -232,43 +247,43 @@ func TestMarkImpressionActed(t *testing.T) {
232 assert.NilError(t, engine.MarkImpressionActed(ctx, "did:test:alice", "feed", "https://a.com/feed"))247 assert.NilError(t, engine.MarkImpressionActed(ctx, "did:test:alice", "feed", "https://a.com/feed"))
233 248
234 var acted bool249 var acted bool
235- assert.NilError(t, database.QueryRowContext(ctx,250+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
236- `SELECT acted FROM recommendation_impressions WHERE user_did = 'did:test:alice' AND target_id = 'https://a.com/feed'`).Scan(&acted))251+ `SELECT acted FROM recs.recommendation_impressions WHERE user_did = 'did:test:alice' AND target_id = 'https://a.com/feed'`).Scan(&acted))
237 assert.Assert(t, acted, "impression should be marked as acted")252 assert.Assert(t, acted, "impression should be marked as acted")
238 }253 }
239 254
240 func TestComputeFollowDistances(t *testing.T) {255 func TestComputeFollowDistances(t *testing.T) {
241 ctx := context.Background()256 ctx := context.Background()
242- database := setupClusterTestDB(t)257+ dbs := setupClusterTestDB(t)
243- seedClusterData(t, ctx, database)258+ seedClusterData(t, ctx, dbs)
244- seedFollowData(t, ctx, database)259+ seedFollowData(t, ctx, dbs)
245 260
246- engine := NewEngine(database.DB, slog.Default())261+ engine := newTestEngine(dbs)
247 assert.NilError(t, engine.ComputeFollowDistances(ctx))262 assert.NilError(t, engine.ComputeFollowDistances(ctx))
248 263
249 var d1, d2 int264 var d1, d2 int
250- assert.NilError(t, database.QueryRowContext(ctx,265+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
251- `SELECT COUNT(*) FROM follow_distances WHERE distance = 1`).Scan(&d1))266+ `SELECT COUNT(*) FROM recs.follow_distances WHERE distance = 1`).Scan(&d1))
252- assert.NilError(t, database.QueryRowContext(ctx,267+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
253- `SELECT COUNT(*) FROM follow_distances WHERE distance = 2`).Scan(&d2))268+ `SELECT COUNT(*) FROM recs.follow_distances WHERE distance = 2`).Scan(&d2))
254 assert.Assert(t, d1 >= 2, "expected at least 2 direct follow distances")269 assert.Assert(t, d1 >= 2, "expected at least 2 direct follow distances")
255 assert.Assert(t, d2 >= 1, "expected at least 1 two-hop distance (alice -> bob -> carol)")270 assert.Assert(t, d2 >= 1, "expected at least 1 two-hop distance (alice -> bob -> carol)")
256 271
257 var dist int272 var dist int
258- assert.NilError(t, database.QueryRowContext(ctx,273+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
259- `SELECT distance FROM follow_distances WHERE user_a = 'did:test:alice' AND user_b = 'did:test:carol'`).Scan(&dist))274+ `SELECT distance FROM recs.follow_distances WHERE user_a = 'did:test:alice' AND user_b = 'did:test:carol'`).Scan(&dist))
260 assert.Equal(t, dist, 2, "alice should be 2 hops from carol")275 assert.Equal(t, dist, 2, "alice should be 2 hops from carol")
261 }276 }
262 277
263 func TestAutoDismissStale(t *testing.T) {278 func TestAutoDismissStale(t *testing.T) {
264 ctx := context.Background()279 ctx := context.Background()
265- database := setupClusterTestDB(t)280+ dbs := setupClusterTestDB(t)
266- seedClusterData(t, ctx, database)281+ seedClusterData(t, ctx, dbs)
267 282
268- engine := NewEngine(database.DB, slog.Default())283+ engine := newTestEngine(dbs)
269 284
270- _, err := database.ExecContext(ctx, `285+ _, err := dbs.Users.ExecContext(ctx, `
271- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)286+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
272 VALUES ('did:test:alice', 'feed', 'https://stale.com/feed', datetime('now', '-31 days'), datetime('now'), 20, 0)287 VALUES ('did:test:alice', 'feed', 'https://stale.com/feed', datetime('now', '-31 days'), datetime('now'), 20, 0)
273 `)288 `)
274 assert.NilError(t, err)289 assert.NilError(t, err)
@@ -282,13 +297,13 @@ func TestAutoDismissStale(t *testing.T) {
282 297
283 func TestAutoDismissStale_DoesNotDismissRecent(t *testing.T) {298 func TestAutoDismissStale_DoesNotDismissRecent(t *testing.T) {
284 ctx := context.Background()299 ctx := context.Background()
285- database := setupClusterTestDB(t)300+ dbs := setupClusterTestDB(t)
286- seedClusterData(t, ctx, database)301+ seedClusterData(t, ctx, dbs)
287 302
288- engine := NewEngine(database.DB, slog.Default())303+ engine := newTestEngine(dbs)
289 304
290- _, err := database.ExecContext(ctx, `305+ _, err := dbs.Users.ExecContext(ctx, `
291- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)306+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
292 VALUES ('did:test:alice', 'feed', 'https://recent.com/feed', datetime('now'), datetime('now'), 5, 0)307 VALUES ('did:test:alice', 'feed', 'https://recent.com/feed', datetime('now'), datetime('now'), 5, 0)
293 `)308 `)
294 assert.NilError(t, err)309 assert.NilError(t, err)
@@ -302,13 +317,13 @@ func TestAutoDismissStale_DoesNotDismissRecent(t *testing.T) {
302 317
303 func TestAutoDismissStale_DoesNotDismissActed(t *testing.T) {318 func TestAutoDismissStale_DoesNotDismissActed(t *testing.T) {
304 ctx := context.Background()319 ctx := context.Background()
305- database := setupClusterTestDB(t)320+ dbs := setupClusterTestDB(t)
306- seedClusterData(t, ctx, database)321+ seedClusterData(t, ctx, dbs)
307 322
308- engine := NewEngine(database.DB, slog.Default())323+ engine := newTestEngine(dbs)
309 324
310- _, err := database.ExecContext(ctx, `325+ _, err := dbs.Users.ExecContext(ctx, `
311- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)326+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
312 VALUES ('did:test:alice', 'feed', 'https://acted.com/feed', datetime('now', '-31 days'), datetime('now'), 20, 1)327 VALUES ('did:test:alice', 'feed', 'https://acted.com/feed', datetime('now', '-31 days'), datetime('now'), 20, 1)
313 `)328 `)
314 assert.NilError(t, err)329 assert.NilError(t, err)
@@ -361,10 +376,10 @@ func TestDiversityFiltering_EmptySiteURL(t *testing.T) {
361 376
362 func TestSignalWeights_Default(t *testing.T) {377 func TestSignalWeights_Default(t *testing.T) {
363 ctx := context.Background()378 ctx := context.Background()
364- database := setupClusterTestDB(t)379+ dbs := setupClusterTestDB(t)
365- seedClusterData(t, ctx, database)380+ seedClusterData(t, ctx, dbs)
366 381
367- engine := NewEngine(database.DB, slog.Default())382+ engine := newTestEngine(dbs)
368 w := engine.GetWeights(ctx, "did:test:alice")383 w := engine.GetWeights(ctx, "did:test:alice")
369 384
370 assert.Equal(t, w.WSub, 1.0)385 assert.Equal(t, w.WSub, 1.0)
@@ -377,19 +392,19 @@ func TestSignalWeights_Default(t *testing.T) {
377 392
378 func TestSignalWeights_RewardPenalize(t *testing.T) {393 func TestSignalWeights_RewardPenalize(t *testing.T) {
379 ctx := context.Background()394 ctx := context.Background()
380- database := setupClusterTestDB(t)395+ dbs := setupClusterTestDB(t)
381- seedClusterData(t, ctx, database)396+ seedClusterData(t, ctx, dbs)
382 397
383- engine := NewEngine(database.DB, slog.Default())398+ engine := newTestEngine(dbs)
384 399
385- _, err := database.ExecContext(ctx, `400+ _, err := dbs.Users.ExecContext(ctx, `
386- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)401+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
387 VALUES ('did:test:alice', 'feed', 'https://a.com/feed', datetime('now'), datetime('now'), 1, 1)402 VALUES ('did:test:alice', 'feed', 'https://a.com/feed', datetime('now'), datetime('now'), 1, 1)
388 `)403 `)
389 assert.NilError(t, err)404 assert.NilError(t, err)
390 for i := range minActionsTune {405 for i := range minActionsTune {
391- _, err = database.ExecContext(ctx, `406+ _, err = dbs.Users.ExecContext(ctx, `
392- INSERT INTO recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)407+ INSERT INTO recs.recommendation_impressions (user_did, target_type, target_id, first_shown_at, last_shown_at, shown_count, acted)
393 VALUES ('did:test:alice', 'feed', ?, datetime('now'), datetime('now'), 1, 1)408 VALUES ('did:test:alice', 'feed', ?, datetime('now'), datetime('now'), 1, 1)
394 `, fmt.Sprintf("https://%d.com/feed", i))409 `, fmt.Sprintf("https://%d.com/feed", i))
395 assert.NilError(t, err)410 assert.NilError(t, err)
@@ -403,19 +418,19 @@ func TestSignalWeights_RewardPenalize(t *testing.T) {
403 418
404 func TestColdStartRecommendations(t *testing.T) {419 func TestColdStartRecommendations(t *testing.T) {
405 ctx := context.Background()420 ctx := context.Background()
406- database := setupClusterTestDB(t)421+ dbs := setupClusterTestDB(t)
407- seedClusterData(t, ctx, database)422+ seedClusterData(t, ctx, dbs)
408- seedFollowData(t, ctx, database)423+ seedFollowData(t, ctx, dbs)
409 424
410- engine := NewEngine(database.DB, slog.Default())425+ engine := newTestEngine(dbs)
411 assert.NilError(t, engine.ComputeFollowDistances(ctx))426 assert.NilError(t, engine.ComputeFollowDistances(ctx))
412 427
413- _, err := database.ExecContext(ctx, `UPDATE feeds SET subscriber_count = 2 WHERE feed_url = 'https://a.com/feed'`)428+ _, err := dbs.Articles.ExecContext(ctx, `UPDATE feeds SET subscriber_count = 2 WHERE feed_url = 'https://a.com/feed'`)
414 assert.NilError(t, err)429 assert.NilError(t, err)
415- _, err = database.ExecContext(ctx, `UPDATE feeds SET subscriber_count = 2 WHERE feed_url = 'https://b.com/feed'`)430+ _, err = dbs.Articles.ExecContext(ctx, `UPDATE feeds SET subscriber_count = 2 WHERE feed_url = 'https://b.com/feed'`)
416 assert.NilError(t, err)431 assert.NilError(t, err)
417 432
418- _, err = database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:newuser", "newuser")433+ _, err = dbs.Users.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:newuser", "newuser")
419 assert.NilError(t, err)434 assert.NilError(t, err)
420 435
421 recs, err := engine.ColdStartRecommendations(ctx, "did:test:newuser", 10)436 recs, err := engine.ColdStartRecommendations(ctx, "did:test:newuser", 10)
@@ -425,11 +440,11 @@ func TestColdStartRecommendations(t *testing.T) {
425 440
426 func TestColdStartRecommendations_NotTriggeredForEstablishedUser(t *testing.T) {441 func TestColdStartRecommendations_NotTriggeredForEstablishedUser(t *testing.T) {
427 ctx := context.Background()442 ctx := context.Background()
428- database := setupClusterTestDB(t)443+ dbs := setupClusterTestDB(t)
429- seedClusterData(t, ctx, database)444+ seedClusterData(t, ctx, dbs)
430- seedFollowData(t, ctx, database)445+ seedFollowData(t, ctx, dbs)
431 446
432- engine := NewEngine(database.DB, slog.Default())447+ engine := newTestEngine(dbs)
433 assert.NilError(t, engine.ComputeFollowDistances(ctx))448 assert.NilError(t, engine.ComputeFollowDistances(ctx))
434 449
435 recs, err := engine.ColdStartRecommendations(ctx, "did:test:alice", 10)450 recs, err := engine.ColdStartRecommendations(ctx, "did:test:alice", 10)
@@ -439,10 +454,10 @@ func TestColdStartRecommendations_NotTriggeredForEstablishedUser(t *testing.T) {
439 454
440 func TestOnDemandPeopleRecommendations(t *testing.T) {455 func TestOnDemandPeopleRecommendations(t *testing.T) {
441 ctx := context.Background()456 ctx := context.Background()
442- database := setupClusterTestDB(t)457+ dbs := setupClusterTestDB(t)
443- seedClusterData(t, ctx, database)458+ seedClusterData(t, ctx, dbs)
444 459
445- engine := NewEngine(database.DB, slog.Default())460+ engine := newTestEngine(dbs)
446 assert.NilError(t, engine.ComputeUserSimilarity(ctx))461 assert.NilError(t, engine.ComputeUserSimilarity(ctx))
447 462
448 recs, err := engine.GetPeopleRecommendations(ctx, "did:test:carol", 10)463 recs, err := engine.GetPeopleRecommendations(ctx, "did:test:carol", 10)
@@ -452,80 +467,80 @@ func TestOnDemandPeopleRecommendations(t *testing.T) {
452 467
453 func TestDismissArticle(t *testing.T) {468 func TestDismissArticle(t *testing.T) {
454 ctx := context.Background()469 ctx := context.Background()
455- database := setupClusterTestDB(t)470+ dbs := setupClusterTestDB(t)
456- seedClusterData(t, ctx, database)471+ seedClusterData(t, ctx, dbs)
457 472
458- engine := NewEngine(database.DB, slog.Default())473+ engine := newTestEngine(dbs)
459 474
460 assert.NilError(t, engine.DismissArticle(ctx, "did:test:alice", "https://a.com/article1", "not_interested"))475 assert.NilError(t, engine.DismissArticle(ctx, "did:test:alice", "https://a.com/article1", "not_interested"))
461 476
462 var count int477 var count int
463- assert.NilError(t, database.QueryRowContext(ctx,478+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
464- `SELECT COUNT(*) FROM dismissed_recommendations WHERE user_did = 'did:test:alice' AND target_type = 'article'`).Scan(&count))479+ `SELECT COUNT(*) FROM recs.dismissed_recommendations WHERE user_did = 'did:test:alice' AND target_type = 'article'`).Scan(&count))
465 assert.Equal(t, count, 1)480 assert.Equal(t, count, 1)
466 }481 }
467 482
468 func TestComputeSignalProfiles(t *testing.T) {483 func TestComputeSignalProfiles(t *testing.T) {
469 ctx := context.Background()484 ctx := context.Background()
470- database := setupClusterTestDB(t)485+ dbs := setupClusterTestDB(t)
471- seedClusterData(t, ctx, database)486+ seedClusterData(t, ctx, dbs)
472 487
473- engine := NewEngine(database.DB, slog.Default())488+ engine := newTestEngine(dbs)
474 assert.NilError(t, engine.ComputeSignalProfiles(ctx))489 assert.NilError(t, engine.ComputeSignalProfiles(ctx))
475 490
476 var count int491 var count int
477- assert.NilError(t, database.QueryRowContext(ctx, `SELECT COUNT(*) FROM user_signal_profiles`).Scan(&count))492+ assert.NilError(t, dbs.Users.QueryRowContext(ctx, `SELECT COUNT(*) FROM recs.user_signal_profiles`).Scan(&count))
478 assert.Assert(t, count >= 3, "expected signal profiles for all users")493 assert.Assert(t, count >= 3, "expected signal profiles for all users")
479 }494 }
480 495
481 func TestDismissFeed_Idempotent(t *testing.T) {496 func TestDismissFeed_Idempotent(t *testing.T) {
482 ctx := context.Background()497 ctx := context.Background()
483- database := setupClusterTestDB(t)498+ dbs := setupClusterTestDB(t)
484- seedClusterData(t, ctx, database)499+ seedClusterData(t, ctx, dbs)
485 500
486- engine := NewEngine(database.DB, slog.Default())501+ engine := newTestEngine(dbs)
487 502
488 assert.NilError(t, engine.DismissFeed(ctx, "did:test:alice", "https://a.com/feed", "reason1"))503 assert.NilError(t, engine.DismissFeed(ctx, "did:test:alice", "https://a.com/feed", "reason1"))
489 assert.NilError(t, engine.DismissFeed(ctx, "did:test:alice", "https://a.com/feed", "reason2"))504 assert.NilError(t, engine.DismissFeed(ctx, "did:test:alice", "https://a.com/feed", "reason2"))
490 505
491 var count int506 var count int
492- assert.NilError(t, database.QueryRowContext(ctx,507+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
493- `SELECT COUNT(*) FROM dismissed_recommendations WHERE user_did = 'did:test:alice' AND target_type = 'feed'`).Scan(&count))508+ `SELECT COUNT(*) FROM recs.dismissed_recommendations WHERE user_did = 'did:test:alice' AND target_type = 'feed'`).Scan(&count))
494 assert.Equal(t, count, 1, "duplicate dismiss should not create extra rows")509 assert.Equal(t, count, 1, "duplicate dismiss should not create extra rows")
495 }510 }
496 511
497 func TestDescriptionBasedFeedSimilarity(t *testing.T) {512 func TestDescriptionBasedFeedSimilarity(t *testing.T) {
498 ctx := context.Background()513 ctx := context.Background()
499- database := setupClusterTestDB(t)514+ dbs := setupClusterTestDB(t)
500 515
501- _, err := database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:alice", "alice")516+ _, err := dbs.Users.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:alice", "alice")
502 assert.NilError(t, err)517 assert.NilError(t, err)
503- _, err = database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:bob", "bob")518+ _, err = dbs.Users.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:bob", "bob")
504 assert.NilError(t, err)519 assert.NilError(t, err)
505 520
506- _, err = database.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type) VALUES (?, ?, ?, ?, 'rss')`,521+ _, err = dbs.Articles.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type) VALUES (?, ?, ?, ?, 'rss')`,
507 "https://go.com/feed", "Go Blog", "https://go.com", "programming language golang software development")522 "https://go.com/feed", "Go Blog", "https://go.com", "programming language golang software development")
508 assert.NilError(t, err)523 assert.NilError(t, err)
509- _, err = database.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type) VALUES (?, ?, ?, ?, 'rss')`,524+ _, err = dbs.Articles.ExecContext(ctx, `INSERT INTO feeds (feed_url, title, site_url, description, feed_type) VALUES (?, ?, ?, ?, 'rss')`,
510 "https://rust.com/feed", "Rust Blog", "https://rust.com", "programming language rust software development")525 "https://rust.com/feed", "Rust Blog", "https://rust.com", "programming language rust software development")
511 assert.NilError(t, err)526 assert.NilError(t, err)
512 527
513- _, err = database.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, "did:test:alice", "https://go.com/feed")528+ _, err = dbs.Articles.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, "did:test:alice", "https://go.com/feed")
514 assert.NilError(t, err)529 assert.NilError(t, err)
515- _, err = database.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, "did:test:bob", "https://rust.com/feed")530+ _, err = dbs.Articles.ExecContext(ctx, `INSERT INTO subscriptions (user_did, feed_url) VALUES (?, ?)`, "did:test:bob", "https://rust.com/feed")
516 assert.NilError(t, err)531 assert.NilError(t, err)
517 532
518- engine := NewEngine(database.DB, slog.Default())533+ engine := newTestEngine(dbs)
519 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))534 assert.NilError(t, engine.ComputeFeedSimilarity(ctx))
520 535
521 var count int536 var count int
522- assert.NilError(t, database.QueryRowContext(ctx, `SELECT COUNT(*) FROM feed_similarity`).Scan(&count))537+ assert.NilError(t, dbs.Users.QueryRowContext(ctx, `SELECT COUNT(*) FROM recs.feed_similarity`).Scan(&count))
523 assert.Assert(t, count >= 0, "description-based similarity should produce pairs")538 assert.Assert(t, count >= 0, "description-based similarity should produce pairs")
524 539
525 if count > 0 {540 if count > 0 {
526 var jaccard float64541 var jaccard float64
527- assert.NilError(t, database.QueryRowContext(ctx,542+ assert.NilError(t, dbs.Users.QueryRowContext(ctx,
528- `SELECT jaccard FROM feed_similarity WHERE feed_a = ? AND feed_b = ?`,543+ `SELECT jaccard FROM recs.feed_similarity WHERE feed_a = ? AND feed_b = ?`,
529 "https://go.com/feed", "https://rust.com/feed").Scan(&jaccard))544 "https://go.com/feed", "https://rust.com/feed").Scan(&jaccard))
530 assert.Assert(t, jaccard > 0, "description word overlap should boost similarity")545 assert.Assert(t, jaccard > 0, "description word overlap should boost similarity")
531 }546 }
modified internal/cluster/scoring.go +1 -1
@@ -428,4 +428,4 @@ func (e *Engine) ColdStartRecommendations(ctx context.Context, userDID string, l
428428 results = append(results, rec)
429429 }
430430 return results, rows.Err()
431-}
\ No newline at end of file
431+}
@@ -428,4 +428,4 @@ func (e *Engine) ColdStartRecommendations(ctx context.Context, userDID string, l
428 results = append(results, rec)428 results = append(results, rec)
429 }429 }
430 return results, rows.Err()430 return results, rows.Err()
431-}
\ No newline at end of file\ No newline at end of file
431+}
modified internal/cluster/social.go +1 -1
@@ -56,4 +56,4 @@ func (e *Engine) ComputeFollowDistancesIncremental(ctx context.Context) error {
5656 }
5757
5858 return e.ComputeFollowDistances(ctx)
59-}
\ No newline at end of file
59+}
@@ -56,4 +56,4 @@ func (e *Engine) ComputeFollowDistancesIncremental(ctx context.Context) error {
56 }56 }
57 57
58 return e.ComputeFollowDistances(ctx)58 return e.ComputeFollowDistances(ctx)
59-}
\ No newline at end of file\ No newline at end of file
59+}
modified internal/cluster/weights.go +1 -1
@@ -90,4 +90,4 @@ func (e *Engine) GetDominantSignal(w SignalWeights) string {
9090 }
9191 }
9292 return best
93-}
\ No newline at end of file
93+}
@@ -90,4 +90,4 @@ func (e *Engine) GetDominantSignal(w SignalWeights) string {
90 }90 }
91 }91 }
92 return best92 return best
93-}
\ No newline at end of file\ No newline at end of file
93+}
added internal/db/batch_test.go +267 -0
new file mode 100644
@@ -0,0 +1,267 @@
1+package db
2+
3+import (
4+ "context"
5+ "testing"
6+ "time"
7+
8+ "gotest.tools/v3/assert"
9+)
10+
11+func TestBatchCreateUsers_InsertsAll(t *testing.T) {
12+ ctx := context.Background()
13+ db := setupTestDB(t)
14+
15+ users := []UserData{
16+ {DID: "did:test:u1", Handle: "user1", DisplayName: "User One", AvatarURL: "https://avatar1.png"},
17+ {DID: "did:test:u2", Handle: "user2", DisplayName: "User Two"},
18+ }
19+ err := db.BatchCreateUsers(ctx, users)
20+ assert.NilError(t, err)
21+
22+ u1, err := db.GetUser(ctx, "did:test:u1")
23+ assert.NilError(t, err)
24+ assert.Equal(t, u1.Handle, "user1")
25+ assert.Equal(t, u1.DisplayName.String, "User One")
26+
27+ u2, err := db.GetUser(ctx, "did:test:u2")
28+ assert.NilError(t, err)
29+ assert.Equal(t, u2.Handle, "user2")
30+ assert.Equal(t, u2.DisplayName.String, "User Two")
31+}
32+
33+func TestBatchCreateUsers_UpsertsExisting(t *testing.T) {
34+ ctx := context.Background()
35+ db := setupTestDB(t)
36+
37+ _, err := db.CreateUser(ctx, "did:test:u1", "old-handle", "", "")
38+ assert.NilError(t, err)
39+
40+ users := []UserData{
41+ {DID: "did:test:u1", Handle: "new-handle", DisplayName: "New Name"},
42+ }
43+ err = db.BatchCreateUsers(ctx, users)
44+ assert.NilError(t, err)
45+
46+ u, err := db.GetUser(ctx, "did:test:u1")
47+ assert.NilError(t, err)
48+ assert.Equal(t, u.Handle, "new-handle")
49+ assert.Equal(t, u.DisplayName.String, "New Name")
50+}
51+
52+func TestBatchCreateUsers_Empty(t *testing.T) {
53+ ctx := context.Background()
54+ db := setupTestDB(t)
55+
56+ err := db.BatchCreateUsers(ctx, nil)
57+ assert.NilError(t, err)
58+}
59+
60+func TestBatchCreateUsers_DoesNotOverwriteWithEmpty(t *testing.T) {
61+ ctx := context.Background()
62+ db := setupTestDB(t)
63+
64+ _, err := db.CreateUser(ctx, "did:test:u1", "handle", "Existing Name", "https://avatar.png")
65+ assert.NilError(t, err)
66+
67+ users := []UserData{
68+ {DID: "did:test:u1", Handle: "", DisplayName: "", AvatarURL: ""},
69+ }
70+ err = db.BatchCreateUsers(ctx, users)
71+ assert.NilError(t, err)
72+
73+ u, err := db.GetUser(ctx, "did:test:u1")
74+ assert.NilError(t, err)
75+ assert.Equal(t, u.Handle, "handle")
76+ assert.Equal(t, u.DisplayName.String, "Existing Name")
77+ assert.Equal(t, u.AvatarURL.String, "https://avatar.png")
78+}
79+
80+func seedSubscriptionData(t *testing.T, ctx context.Context, database *DB) (userDID string) {
81+ t.Helper()
82+ userDID = "did:test:subuser"
83+ _, err := database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, userDID, "subuser")
84+ assert.NilError(t, err)
85+ return userDID
86+}
87+
88+func TestBatchUpsertFeeds_InsertsAll(t *testing.T) {
89+ ctx := context.Background()
90+ db := setupTestDB(t)
91+
92+ feeds := []*Feed{
93+ {FeedURL: "https://a.com/feed.xml", Title: NullStr("Feed A")},
94+ {FeedURL: "https://b.com/feed.xml", Title: NullStr("Feed B")},
95+ }
96+ err := db.BatchUpsertFeeds(ctx, feeds)
97+ assert.NilError(t, err)
98+
99+ f, err := db.GetFeed(ctx, "https://a.com/feed.xml")
100+ assert.NilError(t, err)
101+ assert.Equal(t, f.Title.String, "Feed A")
102+
103+ f, err = db.GetFeed(ctx, "https://b.com/feed.xml")
104+ assert.NilError(t, err)
105+ assert.Equal(t, f.Title.String, "Feed B")
106+}
107+
108+func TestBatchUpsertFeeds_UpdatesExisting(t *testing.T) {
109+ ctx := context.Background()
110+ db := setupTestDB(t)
111+
112+ err := db.UpsertFeed(ctx, &Feed{FeedURL: "https://a.com/feed.xml", Title: NullStr("Old Title")})
113+ assert.NilError(t, err)
114+
115+ feeds := []*Feed{
116+ {FeedURL: "https://a.com/feed.xml", Title: NullStr("New Title")},
117+ }
118+ err = db.BatchUpsertFeeds(ctx, feeds)
119+ assert.NilError(t, err)
120+
121+ f, err := db.GetFeed(ctx, "https://a.com/feed.xml")
122+ assert.NilError(t, err)
123+ assert.Equal(t, f.Title.String, "New Title")
124+}
125+
126+func TestBatchReconcileSubscriptions_CreatesNew(t *testing.T) {
127+ ctx := context.Background()
128+ database := setupTestDB(t)
129+ userDID := seedSubscriptionData(t, ctx, database)
130+
131+ _ = database.UpsertFeed(ctx, &Feed{FeedURL: "https://a.com/feed.xml", Title: NullStr("Feed A")})
132+ _ = database.UpsertFeed(ctx, &Feed{FeedURL: "https://b.com/feed.xml", Title: NullStr("Feed B")})
133+
134+ subs := []SubData{
135+ {FeedURL: "https://a.com/feed.xml", Title: "Feed A", URI: "at://uri1", CID: "cid1"},
136+ {FeedURL: "https://b.com/feed.xml", Title: "Feed B", URI: "at://uri2", CID: "cid2"},
137+ }
138+ err := database.BatchReconcileSubscriptions(ctx, userDID, subs)
139+ assert.NilError(t, err)
140+
141+ subs2, err := database.ListSubscriptions(ctx, userDID, "", 10, 0)
142+ assert.NilError(t, err)
143+ assert.Equal(t, len(subs2), 2)
144+
145+ f, err := database.GetFeed(ctx, "https://a.com/feed.xml")
146+ assert.NilError(t, err)
147+ assert.Equal(t, f.SubscriberCount, 1)
148+}
149+
150+func TestBatchReconcileSubscriptions_BackfillsURI(t *testing.T) {
151+ ctx := context.Background()
152+ database := setupTestDB(t)
153+ userDID := seedSubscriptionData(t, ctx, database)
154+
155+ _ = database.UpsertFeed(ctx, &Feed{FeedURL: "https://a.com/feed.xml"})
156+
157+ err := database.CreateSubscription(ctx, userDID, "https://a.com/feed.xml", "Feed A", "", "", "")
158+ assert.NilError(t, err)
159+
160+ subs := []SubData{
161+ {FeedURL: "https://a.com/feed.xml", URI: "at://new-uri", CID: "new-cid"},
162+ }
163+ err = database.BatchReconcileSubscriptions(ctx, userDID, subs)
164+ assert.NilError(t, err)
165+
166+ s, err := database.GetSubscription(ctx, userDID, "https://a.com/feed.xml")
167+ assert.NilError(t, err)
168+ assert.Equal(t, s.URI.String, "at://new-uri")
169+
170+ f, err := database.GetFeed(ctx, "https://a.com/feed.xml")
171+ assert.NilError(t, err)
172+ assert.Equal(t, f.SubscriberCount, 1)
173+}
174+
175+func TestBatchReconcileSubscriptions_SkipsExistingWithURI(t *testing.T) {
176+ ctx := context.Background()
177+ database := setupTestDB(t)
178+ userDID := seedSubscriptionData(t, ctx, database)
179+
180+ _ = database.UpsertFeed(ctx, &Feed{FeedURL: "https://a.com/feed.xml"})
181+ err := database.CreateSubscription(ctx, userDID, "https://a.com/feed.xml", "Feed A", "", "at://existing", "cid")
182+ assert.NilError(t, err)
183+
184+ subs := []SubData{
185+ {FeedURL: "https://a.com/feed.xml", URI: "at://different", CID: "cid2"},
186+ }
187+ err = database.BatchReconcileSubscriptions(ctx, userDID, subs)
188+ assert.NilError(t, err)
189+
190+ s, err := database.GetSubscription(ctx, userDID, "https://a.com/feed.xml")
191+ assert.NilError(t, err)
192+ assert.Equal(t, s.URI.String, "at://existing")
193+}
194+
195+func TestBatchCreateLikes_InsertsAll(t *testing.T) {
196+ ctx := context.Background()
197+ database := setupTestDB(t)
198+
199+ now := NullTime(time.Now())
200+ likes := []*Like{
201+ {URI: "at://like1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now, CID: NullStr("cid1")},
202+ {URI: "at://like2", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/2", CreatedAt: now, CID: NullStr("cid2")},
203+ }
204+ err := database.BatchCreateLikes(ctx, likes)
205+ assert.NilError(t, err)
206+
207+ exists, err := database.HasLiked(ctx, "did:test:u1", "https://a.com/feed", "https://a.com/1")
208+ assert.NilError(t, err)
209+ assert.Equal(t, exists, true)
210+
211+ exists, err = database.HasLiked(ctx, "did:test:u1", "https://a.com/feed", "https://a.com/2")
212+ assert.NilError(t, err)
213+ assert.Equal(t, exists, true)
214+}
215+
216+func TestBatchCreateLikes_IgnoresDuplicates(t *testing.T) {
217+ ctx := context.Background()
218+ database := setupTestDB(t)
219+
220+ now := NullTime(time.Now())
221+ likes := []*Like{
222+ {URI: "at://like1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now},
223+ }
224+ err := database.BatchCreateLikes(ctx, likes)
225+ assert.NilError(t, err)
226+
227+ likes = append(likes, &Like{URI: "at://like1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now})
228+ err = database.BatchCreateLikes(ctx, likes)
229+ assert.NilError(t, err)
230+}
231+
232+func TestBatchCreateAnnotations_InsertsAll(t *testing.T) {
233+ ctx := context.Background()
234+ database := setupTestDB(t)
235+
236+ now := NullTime(time.Now())
237+ annotations := []*Annotation{
238+ {URI: "at://ann1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", Note: NullStr("Great"), CreatedAt: now},
239+ {URI: "at://ann2", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/2", Note: NullStr("Nice"), CreatedAt: now},
240+ }
241+ err := database.BatchCreateAnnotations(ctx, annotations)
242+ assert.NilError(t, err)
243+
244+ exists, err := database.AnnotationExists(ctx, "at://ann1")
245+ assert.NilError(t, err)
246+ assert.Equal(t, exists, true)
247+
248+ exists, err = database.AnnotationExists(ctx, "at://ann2")
249+ assert.NilError(t, err)
250+ assert.Equal(t, exists, true)
251+}
252+
253+func TestBatchCreateAnnotations_IgnoresDuplicates(t *testing.T) {
254+ ctx := context.Background()
255+ database := setupTestDB(t)
256+
257+ now := NullTime(time.Now())
258+ annotations := []*Annotation{
259+ {URI: "at://ann1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now},
260+ }
261+ err := database.BatchCreateAnnotations(ctx, annotations)
262+ assert.NilError(t, err)
263+
264+ annotations = append(annotations, &Annotation{URI: "at://ann1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now})
265+ err = database.BatchCreateAnnotations(ctx, annotations)
266+ assert.NilError(t, err)
267+}
new file mode 100644
@@ -0,0 +1,267 @@
1+package db
2+
3+import (
4+ "context"
5+ "testing"
6+ "time"
7+
8+ "gotest.tools/v3/assert"
9+)
10+
11+func TestBatchCreateUsers_InsertsAll(t *testing.T) {
12+ ctx := context.Background()
13+ db := setupTestDB(t)
14+
15+ users := []UserData{
16+ {DID: "did:test:u1", Handle: "user1", DisplayName: "User One", AvatarURL: "https://avatar1.png"},
17+ {DID: "did:test:u2", Handle: "user2", DisplayName: "User Two"},
18+ }
19+ err := db.BatchCreateUsers(ctx, users)
20+ assert.NilError(t, err)
21+
22+ u1, err := db.GetUser(ctx, "did:test:u1")
23+ assert.NilError(t, err)
24+ assert.Equal(t, u1.Handle, "user1")
25+ assert.Equal(t, u1.DisplayName.String, "User One")
26+
27+ u2, err := db.GetUser(ctx, "did:test:u2")
28+ assert.NilError(t, err)
29+ assert.Equal(t, u2.Handle, "user2")
30+ assert.Equal(t, u2.DisplayName.String, "User Two")
31+}
32+
33+func TestBatchCreateUsers_UpsertsExisting(t *testing.T) {
34+ ctx := context.Background()
35+ db := setupTestDB(t)
36+
37+ _, err := db.CreateUser(ctx, "did:test:u1", "old-handle", "", "")
38+ assert.NilError(t, err)
39+
40+ users := []UserData{
41+ {DID: "did:test:u1", Handle: "new-handle", DisplayName: "New Name"},
42+ }
43+ err = db.BatchCreateUsers(ctx, users)
44+ assert.NilError(t, err)
45+
46+ u, err := db.GetUser(ctx, "did:test:u1")
47+ assert.NilError(t, err)
48+ assert.Equal(t, u.Handle, "new-handle")
49+ assert.Equal(t, u.DisplayName.String, "New Name")
50+}
51+
52+func TestBatchCreateUsers_Empty(t *testing.T) {
53+ ctx := context.Background()
54+ db := setupTestDB(t)
55+
56+ err := db.BatchCreateUsers(ctx, nil)
57+ assert.NilError(t, err)
58+}
59+
60+func TestBatchCreateUsers_DoesNotOverwriteWithEmpty(t *testing.T) {
61+ ctx := context.Background()
62+ db := setupTestDB(t)
63+
64+ _, err := db.CreateUser(ctx, "did:test:u1", "handle", "Existing Name", "https://avatar.png")
65+ assert.NilError(t, err)
66+
67+ users := []UserData{
68+ {DID: "did:test:u1", Handle: "", DisplayName: "", AvatarURL: ""},
69+ }
70+ err = db.BatchCreateUsers(ctx, users)
71+ assert.NilError(t, err)
72+
73+ u, err := db.GetUser(ctx, "did:test:u1")
74+ assert.NilError(t, err)
75+ assert.Equal(t, u.Handle, "handle")
76+ assert.Equal(t, u.DisplayName.String, "Existing Name")
77+ assert.Equal(t, u.AvatarURL.String, "https://avatar.png")
78+}
79+
80+func seedSubscriptionData(t *testing.T, ctx context.Context, database *DB) (userDID string) {
81+ t.Helper()
82+ userDID = "did:test:subuser"
83+ _, err := database.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, userDID, "subuser")
84+ assert.NilError(t, err)
85+ return userDID
86+}
87+
88+func TestBatchUpsertFeeds_InsertsAll(t *testing.T) {
89+ ctx := context.Background()
90+ db := setupTestDB(t)
91+
92+ feeds := []*Feed{
93+ {FeedURL: "https://a.com/feed.xml", Title: NullStr("Feed A")},
94+ {FeedURL: "https://b.com/feed.xml", Title: NullStr("Feed B")},
95+ }
96+ err := db.BatchUpsertFeeds(ctx, feeds)
97+ assert.NilError(t, err)
98+
99+ f, err := db.GetFeed(ctx, "https://a.com/feed.xml")
100+ assert.NilError(t, err)
101+ assert.Equal(t, f.Title.String, "Feed A")
102+
103+ f, err = db.GetFeed(ctx, "https://b.com/feed.xml")
104+ assert.NilError(t, err)
105+ assert.Equal(t, f.Title.String, "Feed B")
106+}
107+
108+func TestBatchUpsertFeeds_UpdatesExisting(t *testing.T) {
109+ ctx := context.Background()
110+ db := setupTestDB(t)
111+
112+ err := db.UpsertFeed(ctx, &Feed{FeedURL: "https://a.com/feed.xml", Title: NullStr("Old Title")})
113+ assert.NilError(t, err)
114+
115+ feeds := []*Feed{
116+ {FeedURL: "https://a.com/feed.xml", Title: NullStr("New Title")},
117+ }
118+ err = db.BatchUpsertFeeds(ctx, feeds)
119+ assert.NilError(t, err)
120+
121+ f, err := db.GetFeed(ctx, "https://a.com/feed.xml")
122+ assert.NilError(t, err)
123+ assert.Equal(t, f.Title.String, "New Title")
124+}
125+
126+func TestBatchReconcileSubscriptions_CreatesNew(t *testing.T) {
127+ ctx := context.Background()
128+ database := setupTestDB(t)
129+ userDID := seedSubscriptionData(t, ctx, database)
130+
131+ _ = database.UpsertFeed(ctx, &Feed{FeedURL: "https://a.com/feed.xml", Title: NullStr("Feed A")})
132+ _ = database.UpsertFeed(ctx, &Feed{FeedURL: "https://b.com/feed.xml", Title: NullStr("Feed B")})
133+
134+ subs := []SubData{
135+ {FeedURL: "https://a.com/feed.xml", Title: "Feed A", URI: "at://uri1", CID: "cid1"},
136+ {FeedURL: "https://b.com/feed.xml", Title: "Feed B", URI: "at://uri2", CID: "cid2"},
137+ }
138+ err := database.BatchReconcileSubscriptions(ctx, userDID, subs)
139+ assert.NilError(t, err)
140+
141+ subs2, err := database.ListSubscriptions(ctx, userDID, "", 10, 0)
142+ assert.NilError(t, err)
143+ assert.Equal(t, len(subs2), 2)
144+
145+ f, err := database.GetFeed(ctx, "https://a.com/feed.xml")
146+ assert.NilError(t, err)
147+ assert.Equal(t, f.SubscriberCount, 1)
148+}
149+
150+func TestBatchReconcileSubscriptions_BackfillsURI(t *testing.T) {
151+ ctx := context.Background()
152+ database := setupTestDB(t)
153+ userDID := seedSubscriptionData(t, ctx, database)
154+
155+ _ = database.UpsertFeed(ctx, &Feed{FeedURL: "https://a.com/feed.xml"})
156+
157+ err := database.CreateSubscription(ctx, userDID, "https://a.com/feed.xml", "Feed A", "", "", "")
158+ assert.NilError(t, err)
159+
160+ subs := []SubData{
161+ {FeedURL: "https://a.com/feed.xml", URI: "at://new-uri", CID: "new-cid"},
162+ }
163+ err = database.BatchReconcileSubscriptions(ctx, userDID, subs)
164+ assert.NilError(t, err)
165+
166+ s, err := database.GetSubscription(ctx, userDID, "https://a.com/feed.xml")
167+ assert.NilError(t, err)
168+ assert.Equal(t, s.URI.String, "at://new-uri")
169+
170+ f, err := database.GetFeed(ctx, "https://a.com/feed.xml")
171+ assert.NilError(t, err)
172+ assert.Equal(t, f.SubscriberCount, 1)
173+}
174+
175+func TestBatchReconcileSubscriptions_SkipsExistingWithURI(t *testing.T) {
176+ ctx := context.Background()
177+ database := setupTestDB(t)
178+ userDID := seedSubscriptionData(t, ctx, database)
179+
180+ _ = database.UpsertFeed(ctx, &Feed{FeedURL: "https://a.com/feed.xml"})
181+ err := database.CreateSubscription(ctx, userDID, "https://a.com/feed.xml", "Feed A", "", "at://existing", "cid")
182+ assert.NilError(t, err)
183+
184+ subs := []SubData{
185+ {FeedURL: "https://a.com/feed.xml", URI: "at://different", CID: "cid2"},
186+ }
187+ err = database.BatchReconcileSubscriptions(ctx, userDID, subs)
188+ assert.NilError(t, err)
189+
190+ s, err := database.GetSubscription(ctx, userDID, "https://a.com/feed.xml")
191+ assert.NilError(t, err)
192+ assert.Equal(t, s.URI.String, "at://existing")
193+}
194+
195+func TestBatchCreateLikes_InsertsAll(t *testing.T) {
196+ ctx := context.Background()
197+ database := setupTestDB(t)
198+
199+ now := NullTime(time.Now())
200+ likes := []*Like{
201+ {URI: "at://like1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now, CID: NullStr("cid1")},
202+ {URI: "at://like2", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/2", CreatedAt: now, CID: NullStr("cid2")},
203+ }
204+ err := database.BatchCreateLikes(ctx, likes)
205+ assert.NilError(t, err)
206+
207+ exists, err := database.HasLiked(ctx, "did:test:u1", "https://a.com/feed", "https://a.com/1")
208+ assert.NilError(t, err)
209+ assert.Equal(t, exists, true)
210+
211+ exists, err = database.HasLiked(ctx, "did:test:u1", "https://a.com/feed", "https://a.com/2")
212+ assert.NilError(t, err)
213+ assert.Equal(t, exists, true)
214+}
215+
216+func TestBatchCreateLikes_IgnoresDuplicates(t *testing.T) {
217+ ctx := context.Background()
218+ database := setupTestDB(t)
219+
220+ now := NullTime(time.Now())
221+ likes := []*Like{
222+ {URI: "at://like1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now},
223+ }
224+ err := database.BatchCreateLikes(ctx, likes)
225+ assert.NilError(t, err)
226+
227+ likes = append(likes, &Like{URI: "at://like1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now})
228+ err = database.BatchCreateLikes(ctx, likes)
229+ assert.NilError(t, err)
230+}
231+
232+func TestBatchCreateAnnotations_InsertsAll(t *testing.T) {
233+ ctx := context.Background()
234+ database := setupTestDB(t)
235+
236+ now := NullTime(time.Now())
237+ annotations := []*Annotation{
238+ {URI: "at://ann1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", Note: NullStr("Great"), CreatedAt: now},
239+ {URI: "at://ann2", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/2", Note: NullStr("Nice"), CreatedAt: now},
240+ }
241+ err := database.BatchCreateAnnotations(ctx, annotations)
242+ assert.NilError(t, err)
243+
244+ exists, err := database.AnnotationExists(ctx, "at://ann1")
245+ assert.NilError(t, err)
246+ assert.Equal(t, exists, true)
247+
248+ exists, err = database.AnnotationExists(ctx, "at://ann2")
249+ assert.NilError(t, err)
250+ assert.Equal(t, exists, true)
251+}
252+
253+func TestBatchCreateAnnotations_IgnoresDuplicates(t *testing.T) {
254+ ctx := context.Background()
255+ database := setupTestDB(t)
256+
257+ now := NullTime(time.Now())
258+ annotations := []*Annotation{
259+ {URI: "at://ann1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now},
260+ }
261+ err := database.BatchCreateAnnotations(ctx, annotations)
262+ assert.NilError(t, err)
263+
264+ annotations = append(annotations, &Annotation{URI: "at://ann1", AuthorDID: "did:test:u1", FeedURL: "https://a.com/feed", ArticleURL: "https://a.com/1", CreatedAt: now})
265+ err = database.BatchCreateAnnotations(ctx, annotations)
266+ assert.NilError(t, err)
267+}
modified internal/db/feed.go +118 -20
@@ -42,16 +42,7 @@ type Subscription struct {
4242 }
4343
4444 func (db *DB) UpsertFeed(ctx context.Context, feed *Feed) error {
45- _, err := db.ExecContext(ctx, `
46- INSERT INTO feeds (feed_url, title, site_url, description, feed_type)
47- VALUES (?, ?, ?, ?, ?)
48- ON CONFLICT(feed_url) DO UPDATE SET
49- title = excluded.title,
50- site_url = excluded.site_url,
51- description = excluded.description,
52- feed_type = excluded.feed_type
53- `, feed.FeedURL, feed.Title, feed.SiteURL, feed.Description, feed.FeedType)
54- return err
45+ return db.BatchUpsertFeeds(ctx, []*Feed{feed})
5546 }
5647
5748 func (db *DB) GetFeed(ctx context.Context, feedURL string) (*Feed, error) {
@@ -138,18 +129,14 @@ func (db *DB) DecrementSubscriberCount(ctx context.Context, feedURL string) erro
138129 }
139130
140131 func (db *DB) CreateSubscription(ctx context.Context, userDID, feedURL, title, category, uri, cid string) error {
141- result, err := db.ExecContext(ctx, `
142- INSERT OR IGNORE INTO subscriptions (user_did, feed_url, title, category, uri, cid)
143- VALUES (?, ?, ?, ?, ?, ?)
144- `, userDID, feedURL, nilIfEmpty(title), category, uriOrNil(category, uri), uriOrNil(category, cid))
145- if err != nil {
146- return err
147- }
148- n, _ := result.RowsAffected()
149- if n == 0 {
132+ existing, err := db.GetSubscription(ctx, userDID, feedURL)
133+ if err == nil && existing != nil {
134+ if !existing.URI.Valid || existing.URI.String == "" {
135+ return db.UpdateSubscriptionURI(ctx, userDID, feedURL, uri, cid)
136+ }
150137 return ErrDuplicateSubscription
151138 }
152- return db.IncrementSubscriberCount(ctx, feedURL)
139+ return db.BatchReconcileSubscriptions(ctx, userDID, []SubData{{FeedURL: feedURL, Title: title, Category: category, URI: uri, CID: cid}})
153140 }
154141
155142 func (db *DB) UpdateSubscriptionURI(ctx context.Context, userDID, feedURL, uri, cid string) error {
@@ -382,6 +369,117 @@ func (db *DB) ListAllFeeds(ctx context.Context, limit, offset int) ([]*Feed, err
382369 return feeds, rows.Err()
383370 }
384371
372+type SubData struct {
373+ FeedURL string
374+ Title string
375+ Category string
376+ URI string
377+ CID string
378+}
379+
380+func (db *DB) BatchUpsertFeeds(ctx context.Context, feeds []*Feed) error {
381+ if len(feeds) == 0 {
382+ return nil
383+ }
384+ tx, err := db.BeginTx(ctx, nil)
385+ if err != nil {
386+ return err
387+ }
388+ defer tx.Rollback()
389+
390+ stmt, err := tx.PrepareContext(ctx, `
391+ INSERT INTO feeds (feed_url, title, site_url, description, feed_type)
392+ VALUES (?, ?, ?, ?, ?)
393+ ON CONFLICT(feed_url) DO UPDATE SET
394+ title = excluded.title,
395+ site_url = excluded.site_url,
396+ description = excluded.description,
397+ feed_type = excluded.feed_type
398+ `)
399+ if err != nil {
400+ return err
401+ }
402+ defer stmt.Close()
403+
404+ for _, f := range feeds {
405+ if _, err := stmt.ExecContext(ctx, f.FeedURL, f.Title, f.SiteURL, f.Description, f.FeedType); err != nil {
406+ return err
407+ }
408+ }
409+ return tx.Commit()
410+}
411+
412+func (db *DB) BatchReconcileSubscriptions(ctx context.Context, userDID string, subs []SubData) error {
413+ if len(subs) == 0 {
414+ return nil
415+ }
416+ tx, err := db.BeginTx(ctx, nil)
417+ if err != nil {
418+ return err
419+ }
420+ defer tx.Rollback()
421+
422+ rows, err := tx.QueryContext(ctx, `SELECT feed_url, COALESCE(uri, '') FROM subscriptions WHERE user_did = ?`, userDID)
423+ if err != nil {
424+ return err
425+ }
426+ existing := make(map[string]string, len(subs))
427+ for rows.Next() {
428+ var feedURL, uri string
429+ if err := rows.Scan(&feedURL, &uri); err != nil {
430+ rows.Close()
431+ return err
432+ }
433+ existing[feedURL] = uri
434+ }
435+ rows.Close()
436+
437+ insertStmt, err := tx.PrepareContext(ctx, `
438+ INSERT OR IGNORE INTO subscriptions (user_did, feed_url, title, category, uri, cid)
439+ VALUES (?, ?, ?, ?, ?, ?)
440+ `)
441+ if err != nil {
442+ return err
443+ }
444+ defer insertStmt.Close()
445+
446+ updateStmt, err := tx.PrepareContext(ctx, `
447+ UPDATE subscriptions SET uri = ?, cid = ? WHERE user_did = ? AND feed_url = ?
448+ `)
449+ if err != nil {
450+ return err
451+ }
452+ defer updateStmt.Close()
453+
454+ incrStmt, err := tx.PrepareContext(ctx, `UPDATE feeds SET subscriber_count = subscriber_count + 1 WHERE feed_url = ?`)
455+ if err != nil {
456+ return err
457+ }
458+ defer incrStmt.Close()
459+
460+ for _, sub := range subs {
461+ if existingURI, ok := existing[sub.FeedURL]; ok {
462+ if existingURI == "" && sub.URI != "" {
463+ if _, err := updateStmt.ExecContext(ctx, sub.URI, sub.CID, userDID, sub.FeedURL); err != nil {
464+ return err
465+ }
466+ }
467+ continue
468+ }
469+ result, err := insertStmt.ExecContext(ctx, userDID, sub.FeedURL, nilIfEmpty(sub.Title), sub.Category, uriOrNil(sub.Category, sub.URI), uriOrNil(sub.Category, sub.CID))
470+ if err != nil {
471+ return err
472+ }
473+ n, _ := result.RowsAffected()
474+ if n > 0 {
475+ if _, err := incrStmt.ExecContext(ctx, sub.FeedURL); err != nil {
476+ return err
477+ }
478+ }
479+ }
480+ return tx.Commit()
481+}
482+
385483 func (db *DB) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) {
386484 rows, err := db.QueryContext(ctx, `
387485 SELECT feed_url, title, site_url, description, feed_type,
@@ -42,16 +42,7 @@ type Subscription struct {
42 }42 }
43 43
44 func (db *DB) UpsertFeed(ctx context.Context, feed *Feed) error {44 func (db *DB) UpsertFeed(ctx context.Context, feed *Feed) error {
45- _, err := db.ExecContext(ctx, `45+ return db.BatchUpsertFeeds(ctx, []*Feed{feed})
46- INSERT INTO feeds (feed_url, title, site_url, description, feed_type)
47- VALUES (?, ?, ?, ?, ?)
48- ON CONFLICT(feed_url) DO UPDATE SET
49- title = excluded.title,
50- site_url = excluded.site_url,
51- description = excluded.description,
52- feed_type = excluded.feed_type
53- `, feed.FeedURL, feed.Title, feed.SiteURL, feed.Description, feed.FeedType)
54- return err
55 }46 }
56 47
57 func (db *DB) GetFeed(ctx context.Context, feedURL string) (*Feed, error) {48 func (db *DB) GetFeed(ctx context.Context, feedURL string) (*Feed, error) {
@@ -138,18 +129,14 @@ func (db *DB) DecrementSubscriberCount(ctx context.Context, feedURL string) erro
138 }129 }
139 130
140 func (db *DB) CreateSubscription(ctx context.Context, userDID, feedURL, title, category, uri, cid string) error {131 func (db *DB) CreateSubscription(ctx context.Context, userDID, feedURL, title, category, uri, cid string) error {
141- result, err := db.ExecContext(ctx, `132+ existing, err := db.GetSubscription(ctx, userDID, feedURL)
142- INSERT OR IGNORE INTO subscriptions (user_did, feed_url, title, category, uri, cid)133+ if err == nil && existing != nil {
143- VALUES (?, ?, ?, ?, ?, ?)134+ if !existing.URI.Valid || existing.URI.String == "" {
144- `, userDID, feedURL, nilIfEmpty(title), category, uriOrNil(category, uri), uriOrNil(category, cid))135+ return db.UpdateSubscriptionURI(ctx, userDID, feedURL, uri, cid)
145- if err != nil {136+ }
146- return err
147- }
148- n, _ := result.RowsAffected()
149- if n == 0 {
150 return ErrDuplicateSubscription137 return ErrDuplicateSubscription
151 }138 }
152- return db.IncrementSubscriberCount(ctx, feedURL)139+ return db.BatchReconcileSubscriptions(ctx, userDID, []SubData{{FeedURL: feedURL, Title: title, Category: category, URI: uri, CID: cid}})
153 }140 }
154 141
155 func (db *DB) UpdateSubscriptionURI(ctx context.Context, userDID, feedURL, uri, cid string) error {142 func (db *DB) UpdateSubscriptionURI(ctx context.Context, userDID, feedURL, uri, cid string) error {
@@ -382,6 +369,117 @@ func (db *DB) ListAllFeeds(ctx context.Context, limit, offset int) ([]*Feed, err
382 return feeds, rows.Err()369 return feeds, rows.Err()
383 }370 }
384 371
372+type SubData struct {
373+ FeedURL string
374+ Title string
375+ Category string
376+ URI string
377+ CID string
378+}
379+
380+func (db *DB) BatchUpsertFeeds(ctx context.Context, feeds []*Feed) error {
381+ if len(feeds) == 0 {
382+ return nil
383+ }
384+ tx, err := db.BeginTx(ctx, nil)
385+ if err != nil {
386+ return err
387+ }
388+ defer tx.Rollback()
389+
390+ stmt, err := tx.PrepareContext(ctx, `
391+ INSERT INTO feeds (feed_url, title, site_url, description, feed_type)
392+ VALUES (?, ?, ?, ?, ?)
393+ ON CONFLICT(feed_url) DO UPDATE SET
394+ title = excluded.title,
395+ site_url = excluded.site_url,
396+ description = excluded.description,
397+ feed_type = excluded.feed_type
398+ `)
399+ if err != nil {
400+ return err
401+ }
402+ defer stmt.Close()
403+
404+ for _, f := range feeds {
405+ if _, err := stmt.ExecContext(ctx, f.FeedURL, f.Title, f.SiteURL, f.Description, f.FeedType); err != nil {
406+ return err
407+ }
408+ }
409+ return tx.Commit()
410+}
411+
412+func (db *DB) BatchReconcileSubscriptions(ctx context.Context, userDID string, subs []SubData) error {
413+ if len(subs) == 0 {
414+ return nil
415+ }
416+ tx, err := db.BeginTx(ctx, nil)
417+ if err != nil {
418+ return err
419+ }
420+ defer tx.Rollback()
421+
422+ rows, err := tx.QueryContext(ctx, `SELECT feed_url, COALESCE(uri, '') FROM subscriptions WHERE user_did = ?`, userDID)
423+ if err != nil {
424+ return err
425+ }
426+ existing := make(map[string]string, len(subs))
427+ for rows.Next() {
428+ var feedURL, uri string
429+ if err := rows.Scan(&feedURL, &uri); err != nil {
430+ rows.Close()
431+ return err
432+ }
433+ existing[feedURL] = uri
434+ }
435+ rows.Close()
436+
437+ insertStmt, err := tx.PrepareContext(ctx, `
438+ INSERT OR IGNORE INTO subscriptions (user_did, feed_url, title, category, uri, cid)
439+ VALUES (?, ?, ?, ?, ?, ?)
440+ `)
441+ if err != nil {
442+ return err
443+ }
444+ defer insertStmt.Close()
445+
446+ updateStmt, err := tx.PrepareContext(ctx, `
447+ UPDATE subscriptions SET uri = ?, cid = ? WHERE user_did = ? AND feed_url = ?
448+ `)
449+ if err != nil {
450+ return err
451+ }
452+ defer updateStmt.Close()
453+
454+ incrStmt, err := tx.PrepareContext(ctx, `UPDATE feeds SET subscriber_count = subscriber_count + 1 WHERE feed_url = ?`)
455+ if err != nil {
456+ return err
457+ }
458+ defer incrStmt.Close()
459+
460+ for _, sub := range subs {
461+ if existingURI, ok := existing[sub.FeedURL]; ok {
462+ if existingURI == "" && sub.URI != "" {
463+ if _, err := updateStmt.ExecContext(ctx, sub.URI, sub.CID, userDID, sub.FeedURL); err != nil {
464+ return err
465+ }
466+ }
467+ continue
468+ }
469+ result, err := insertStmt.ExecContext(ctx, userDID, sub.FeedURL, nilIfEmpty(sub.Title), sub.Category, uriOrNil(sub.Category, sub.URI), uriOrNil(sub.Category, sub.CID))
470+ if err != nil {
471+ return err
472+ }
473+ n, _ := result.RowsAffected()
474+ if n > 0 {
475+ if _, err := incrStmt.ExecContext(ctx, sub.FeedURL); err != nil {
476+ return err
477+ }
478+ }
479+ }
480+ return tx.Commit()
481+}
482+
385 func (db *DB) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) {483 func (db *DB) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) {
386 rows, err := db.QueryContext(ctx, `484 rows, err := db.QueryContext(ctx, `
387 SELECT feed_url, title, site_url, description, feed_type,485 SELECT feed_url, title, site_url, description, feed_type,
modified internal/db/social.go +57 -11
@@ -36,11 +36,7 @@ type Like struct {
3636 }
3737
3838 func (db *DB) CreateAnnotation(ctx context.Context, a *Annotation) error {
39- _, err := db.ExecContext(ctx, `
40- INSERT OR IGNORE INTO annotations (uri, author_did, feed_url, article_url, quote, note, tags, rating, created_at, cid)
41- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
42- `, a.URI, a.AuthorDID, a.FeedURL, a.ArticleURL, a.Quote, a.Note, a.Tags, a.Rating, a.CreatedAt, a.CID)
43- return err
39+ return db.BatchCreateAnnotations(ctx, []*Annotation{a})
4440 }
4541
4642 func (db *DB) GetAnnotation(ctx context.Context, id int64) (*Annotation, error) {
@@ -121,19 +117,69 @@ func (db *DB) ListAnnotations(ctx context.Context, feedURL, articleURL, authorDI
121117 return annotations, rows.Err()
122118 }
123119
124-func (db *DB) CreateLike(ctx context.Context, l *Like) error {
125- result, err := db.ExecContext(ctx, `
120+func (db *DB) BatchCreateLikes(ctx context.Context, likes []*Like) error {
121+ if len(likes) == 0 {
122+ return nil
123+ }
124+ tx, err := db.BeginTx(ctx, nil)
125+ if err != nil {
126+ return err
127+ }
128+ defer tx.Rollback()
129+
130+ stmt, err := tx.PrepareContext(ctx, `
126131 INSERT OR IGNORE INTO likes (uri, author_did, feed_url, article_url, created_at, cid)
127132 VALUES (?, ?, ?, ?, ?, ?)
128- `, l.URI, l.AuthorDID, l.FeedURL, l.ArticleURL, l.CreatedAt, l.CID)
133+ `)
134+ if err != nil {
135+ return err
136+ }
137+ defer stmt.Close()
138+
139+ for _, l := range likes {
140+ if _, err := stmt.ExecContext(ctx, l.URI, l.AuthorDID, l.FeedURL, l.ArticleURL, l.CreatedAt, l.CID); err != nil {
141+ return err
142+ }
143+ }
144+ return tx.Commit()
145+}
146+
147+func (db *DB) BatchCreateAnnotations(ctx context.Context, annotations []*Annotation) error {
148+ if len(annotations) == 0 {
149+ return nil
150+ }
151+ tx, err := db.BeginTx(ctx, nil)
152+ if err != nil {
153+ return err
154+ }
155+ defer tx.Rollback()
156+
157+ stmt, err := tx.PrepareContext(ctx, `
158+ INSERT OR IGNORE INTO annotations (uri, author_did, feed_url, article_url, quote, note, tags, rating, created_at, cid)
159+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
160+ `)
161+ if err != nil {
162+ return err
163+ }
164+ defer stmt.Close()
165+
166+ for _, a := range annotations {
167+ if _, err := stmt.ExecContext(ctx, a.URI, a.AuthorDID, a.FeedURL, a.ArticleURL, a.Quote, a.Note, a.Tags, a.Rating, a.CreatedAt, a.CID); err != nil {
168+ return err
169+ }
170+ }
171+ return tx.Commit()
172+}
173+
174+func (db *DB) CreateLike(ctx context.Context, l *Like) error {
175+ exists, err := db.HasLiked(ctx, l.AuthorDID, l.FeedURL, l.ArticleURL)
129176 if err != nil {
130177 return err
131178 }
132- n, _ := result.RowsAffected()
133- if n == 0 {
179+ if exists {
134180 return ErrDuplicateLike
135181 }
136- return nil
182+ return db.BatchCreateLikes(ctx, []*Like{l})
137183 }
138184
139185 func (db *DB) DeleteLike(ctx context.Context, uri string) error {
@@ -36,11 +36,7 @@ type Like struct {
36 }36 }
37 37
38 func (db *DB) CreateAnnotation(ctx context.Context, a *Annotation) error {38 func (db *DB) CreateAnnotation(ctx context.Context, a *Annotation) error {
39- _, err := db.ExecContext(ctx, `39+ return db.BatchCreateAnnotations(ctx, []*Annotation{a})
40- INSERT OR IGNORE INTO annotations (uri, author_did, feed_url, article_url, quote, note, tags, rating, created_at, cid)
41- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
42- `, a.URI, a.AuthorDID, a.FeedURL, a.ArticleURL, a.Quote, a.Note, a.Tags, a.Rating, a.CreatedAt, a.CID)
43- return err
44 }40 }
45 41
46 func (db *DB) GetAnnotation(ctx context.Context, id int64) (*Annotation, error) {42 func (db *DB) GetAnnotation(ctx context.Context, id int64) (*Annotation, error) {
@@ -121,19 +117,69 @@ func (db *DB) ListAnnotations(ctx context.Context, feedURL, articleURL, authorDI
121 return annotations, rows.Err()117 return annotations, rows.Err()
122 }118 }
123 119
124-func (db *DB) CreateLike(ctx context.Context, l *Like) error {120+func (db *DB) BatchCreateLikes(ctx context.Context, likes []*Like) error {
125- result, err := db.ExecContext(ctx, `121+ if len(likes) == 0 {
122+ return nil
123+ }
124+ tx, err := db.BeginTx(ctx, nil)
125+ if err != nil {
126+ return err
127+ }
128+ defer tx.Rollback()
129+
130+ stmt, err := tx.PrepareContext(ctx, `
126 INSERT OR IGNORE INTO likes (uri, author_did, feed_url, article_url, created_at, cid)131 INSERT OR IGNORE INTO likes (uri, author_did, feed_url, article_url, created_at, cid)
127 VALUES (?, ?, ?, ?, ?, ?)132 VALUES (?, ?, ?, ?, ?, ?)
128- `, l.URI, l.AuthorDID, l.FeedURL, l.ArticleURL, l.CreatedAt, l.CID)133+ `)
134+ if err != nil {
135+ return err
136+ }
137+ defer stmt.Close()
138+
139+ for _, l := range likes {
140+ if _, err := stmt.ExecContext(ctx, l.URI, l.AuthorDID, l.FeedURL, l.ArticleURL, l.CreatedAt, l.CID); err != nil {
141+ return err
142+ }
143+ }
144+ return tx.Commit()
145+}
146+
147+func (db *DB) BatchCreateAnnotations(ctx context.Context, annotations []*Annotation) error {
148+ if len(annotations) == 0 {
149+ return nil
150+ }
151+ tx, err := db.BeginTx(ctx, nil)
152+ if err != nil {
153+ return err
154+ }
155+ defer tx.Rollback()
156+
157+ stmt, err := tx.PrepareContext(ctx, `
158+ INSERT OR IGNORE INTO annotations (uri, author_did, feed_url, article_url, quote, note, tags, rating, created_at, cid)
159+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
160+ `)
161+ if err != nil {
162+ return err
163+ }
164+ defer stmt.Close()
165+
166+ for _, a := range annotations {
167+ if _, err := stmt.ExecContext(ctx, a.URI, a.AuthorDID, a.FeedURL, a.ArticleURL, a.Quote, a.Note, a.Tags, a.Rating, a.CreatedAt, a.CID); err != nil {
168+ return err
169+ }
170+ }
171+ return tx.Commit()
172+}
173+
174+func (db *DB) CreateLike(ctx context.Context, l *Like) error {
175+ exists, err := db.HasLiked(ctx, l.AuthorDID, l.FeedURL, l.ArticleURL)
129 if err != nil {176 if err != nil {
130 return err177 return err
131 }178 }
132- n, _ := result.RowsAffected()179+ if exists {
133- if n == 0 {
134 return ErrDuplicateLike180 return ErrDuplicateLike
135 }181 }
136- return nil182+ return db.BatchCreateLikes(ctx, []*Like{l})
137 }183 }
138 184
139 func (db *DB) DeleteLike(ctx context.Context, uri string) error {185 func (db *DB) DeleteLike(ctx context.Context, uri string) error {
modified internal/db/user.go +34 -3
@@ -14,8 +14,24 @@ type User struct {
1414 UpdatedAt sql.NullTime
1515 }
1616
17-func (db *DB) CreateUser(ctx context.Context, did, handle, displayName, avatarURL string) (*User, error) {
18- _, err := db.ExecContext(ctx, `
17+type UserData struct {
18+ DID string
19+ Handle string
20+ DisplayName string
21+ AvatarURL string
22+}
23+
24+func (db *DB) BatchCreateUsers(ctx context.Context, users []UserData) error {
25+ if len(users) == 0 {
26+ return nil
27+ }
28+ tx, err := db.BeginTx(ctx, nil)
29+ if err != nil {
30+ return err
31+ }
32+ defer tx.Rollback()
33+
34+ stmt, err := tx.PrepareContext(ctx, `
1935 INSERT INTO users (did, handle, display_name, avatar_url, updated_at)
2036 VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
2137 ON CONFLICT(did) DO UPDATE SET
@@ -23,7 +39,22 @@ func (db *DB) CreateUser(ctx context.Context, did, handle, displayName, avatarUR
2339 display_name = COALESCE(NULLIF(excluded.display_name, ''), users.display_name),
2440 avatar_url = COALESCE(NULLIF(excluded.avatar_url, ''), users.avatar_url),
2541 updated_at = CURRENT_TIMESTAMP
26- `, did, handle, displayName, avatarURL)
42+ `)
43+ if err != nil {
44+ return err
45+ }
46+ defer stmt.Close()
47+
48+ for _, u := range users {
49+ if _, err := stmt.ExecContext(ctx, u.DID, u.Handle, u.DisplayName, u.AvatarURL); err != nil {
50+ return err
51+ }
52+ }
53+ return tx.Commit()
54+}
55+
56+func (db *DB) CreateUser(ctx context.Context, did, handle, displayName, avatarURL string) (*User, error) {
57+ err := db.BatchCreateUsers(ctx, []UserData{{DID: did, Handle: handle, DisplayName: displayName, AvatarURL: avatarURL}})
2758 if err != nil {
2859 return nil, err
2960 }
@@ -14,8 +14,24 @@ type User struct {
14 UpdatedAt sql.NullTime14 UpdatedAt sql.NullTime
15 }15 }
16 16
17-func (db *DB) CreateUser(ctx context.Context, did, handle, displayName, avatarURL string) (*User, error) {17+type UserData struct {
18- _, err := db.ExecContext(ctx, `18+ DID string
19+ Handle string
20+ DisplayName string
21+ AvatarURL string
22+}
23+
24+func (db *DB) BatchCreateUsers(ctx context.Context, users []UserData) error {
25+ if len(users) == 0 {
26+ return nil
27+ }
28+ tx, err := db.BeginTx(ctx, nil)
29+ if err != nil {
30+ return err
31+ }
32+ defer tx.Rollback()
33+
34+ stmt, err := tx.PrepareContext(ctx, `
19 INSERT INTO users (did, handle, display_name, avatar_url, updated_at)35 INSERT INTO users (did, handle, display_name, avatar_url, updated_at)
20 VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)36 VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
21 ON CONFLICT(did) DO UPDATE SET37 ON CONFLICT(did) DO UPDATE SET
@@ -23,7 +39,22 @@ func (db *DB) CreateUser(ctx context.Context, did, handle, displayName, avatarUR
23 display_name = COALESCE(NULLIF(excluded.display_name, ''), users.display_name),39 display_name = COALESCE(NULLIF(excluded.display_name, ''), users.display_name),
24 avatar_url = COALESCE(NULLIF(excluded.avatar_url, ''), users.avatar_url),40 avatar_url = COALESCE(NULLIF(excluded.avatar_url, ''), users.avatar_url),
25 updated_at = CURRENT_TIMESTAMP41 updated_at = CURRENT_TIMESTAMP
26- `, did, handle, displayName, avatarURL)42+ `)
43+ if err != nil {
44+ return err
45+ }
46+ defer stmt.Close()
47+
48+ for _, u := range users {
49+ if _, err := stmt.ExecContext(ctx, u.DID, u.Handle, u.DisplayName, u.AvatarURL); err != nil {
50+ return err
51+ }
52+ }
53+ return tx.Commit()
54+}
55+
56+func (db *DB) CreateUser(ctx context.Context, did, handle, displayName, avatarURL string) (*User, error) {
57+ err := db.BatchCreateUsers(ctx, []UserData{{DID: did, Handle: handle, DisplayName: displayName, AvatarURL: avatarURL}})
27 if err != nil {58 if err != nil {
28 return nil, err59 return nil, err
29 }60 }
modified internal/server/server.go +19 -9
@@ -12,6 +12,8 @@ import (
1212 "strconv"
1313 "strings"
1414 "sync"
15+
16+ "golang.org/x/sync/errgroup"
1517 "time"
1618
1719 "github.com/go-chi/chi/v5"
@@ -488,16 +490,24 @@ func (s *Server) BackfillFromCollectionDir(ctx context.Context, collectionDirURL
488490 defer func() { <-sem }()
489491 defer wg.Done()
490492
491- handle := did
492- if ident, err := atproto.ResolveIdentity(ctx, did); err == nil {
493- handle = ident.Handle.String()
494- }
493+ g, gCtx := errgroup.WithContext(ctx)
495494
496- var displayName, avatarURL string
497- if _, dn, avatar, err := atproto.FetchProfile(ctx, did); err == nil {
498- displayName = dn
499- avatarURL = avatar
500- }
495+ var handle, displayName, avatarURL string
496+ g.Go(func() error {
497+ if ident, err := atproto.ResolveIdentity(gCtx, did); err == nil {
498+ handle = ident.Handle.String()
499+ }
500+ return nil
501+ })
502+ g.Go(func() error {
503+ if h, dn, avatar, err := atproto.FetchProfile(gCtx, did); err == nil {
504+ handle = h
505+ displayName = dn
506+ avatarURL = avatar
507+ }
508+ return nil
509+ })
510+ _ = g.Wait()
501511
502512 if _, err := s.dbs.Users.CreateUser(ctx, did, handle, displayName, avatarURL); err != nil {
503513 s.logger.Error("failed to create user during backfill", "error", err, "did", did)
@@ -12,6 +12,8 @@ import (
12 "strconv"12 "strconv"
13 "strings"13 "strings"
14 "sync"14 "sync"
15+
16+ "golang.org/x/sync/errgroup"
15 "time"17 "time"
16 18
17 "github.com/go-chi/chi/v5"19 "github.com/go-chi/chi/v5"
@@ -488,16 +490,24 @@ func (s *Server) BackfillFromCollectionDir(ctx context.Context, collectionDirURL
488 defer func() { <-sem }()490 defer func() { <-sem }()
489 defer wg.Done()491 defer wg.Done()
490 492
491- handle := did493+ g, gCtx := errgroup.WithContext(ctx)
492- if ident, err := atproto.ResolveIdentity(ctx, did); err == nil {
493- handle = ident.Handle.String()
494- }
495 494
496- var displayName, avatarURL string495+ var handle, displayName, avatarURL string
497- if _, dn, avatar, err := atproto.FetchProfile(ctx, did); err == nil {496+ g.Go(func() error {
498- displayName = dn497+ if ident, err := atproto.ResolveIdentity(gCtx, did); err == nil {
499- avatarURL = avatar498+ handle = ident.Handle.String()
500- }499+ }
500+ return nil
501+ })
502+ g.Go(func() error {
503+ if h, dn, avatar, err := atproto.FetchProfile(gCtx, did); err == nil {
504+ handle = h
505+ displayName = dn
506+ avatarURL = avatar
507+ }
508+ return nil
509+ })
510+ _ = g.Wait()
501 511
502 if _, err := s.dbs.Users.CreateUser(ctx, did, handle, displayName, avatarURL); err != nil {512 if _, err := s.dbs.Users.CreateUser(ctx, did, handle, displayName, avatarURL); err != nil {
503 s.logger.Error("failed to create user during backfill", "error", err, "did", did)513 s.logger.Error("failed to create user during backfill", "error", err, "did", did)