rickub
nandi/gleanpublic⑂ Fork 0
⑂ fd9ed81
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Optimize SQL similarity queries using temp tablesUnverified

Julien Robert committed 2026-04-21T16:42:43+02:00 Browse files
fd9ed81 parent: 097e22b
modified internal/cluster/jaccard.go +212 -124
@@ -68,7 +68,6 @@ func (e *Engine) ComputeArticleRecommendations(ctx context.Context) error {
6868 SELECT 1 FROM likes ul WHERE ul.author_did = targets.target AND ul.feed_url = l.feed_url AND ul.article_url = l.article_url
6969 )
7070 GROUP BY targets.target, l.feed_url, l.article_url
71- HAVING COUNT(*) > 0
7271 ORDER BY score DESC
7372 `, e.config.SimilarityThreshold, e.config.SimilarityThreshold)
7473
@@ -117,7 +116,6 @@ func (e *Engine) ComputeFeedSimilarity(ctx context.Context) error {
117116 JOIN feeds f1 ON f1.feed_url = s1.feed_url
118117 JOIN feeds f2 ON f2.feed_url = s2.feed_url
119118 GROUP BY s1.feed_url, s2.feed_url
120- HAVING COUNT(*) > 0
121119 `)
122120 if err != nil {
123121 return err
@@ -160,65 +158,66 @@ func (e *Engine) computeDescriptionSimilarity(ctx context.Context, tx *sql.Tx) e
160158 return err
161159 }
162160
163- descUpdate := fmt.Sprintf(`
164- UPDATE feed_similarity SET
165- jaccard = jaccard + %g * CAST(word_overlap.common AS REAL) / NULLIF(
166- (SELECT COUNT(DISTINCT word) FROM _feed_words WHERE feed_url = feed_similarity.feed_a) +
167- (SELECT COUNT(DISTINCT word) FROM _feed_words WHERE feed_url = feed_similarity.feed_b) -
168- CAST(word_overlap.common AS REAL),
169- 0
170- )
171- FROM (
172- SELECT w1.feed_url AS feed_a, w2.feed_url AS feed_b, COUNT(DISTINCT w1.word) AS common
173- FROM _feed_words w1
174- JOIN _feed_words w2 ON w1.word = w2.word AND w1.feed_url < w2.feed_url
175- GROUP BY w1.feed_url, w2.feed_url
176- HAVING common > 1
177- ) word_overlap
178- WHERE feed_similarity.feed_a = word_overlap.feed_a
179- AND feed_similarity.feed_b = word_overlap.feed_b
180- `, e.config.DescriptionWeight)
161+ if _, err := tx.ExecContext(ctx, `
162+ CREATE TEMP TABLE IF NOT EXISTS _feed_word_counts (feed_url TEXT PRIMARY KEY, cnt INT)
163+ `); err != nil {
164+ return err
165+ }
166+ if _, err := tx.ExecContext(ctx, `DELETE FROM _feed_word_counts`); err != nil {
167+ return err
168+ }
169+ if _, err := tx.ExecContext(ctx, `
170+ INSERT INTO _feed_word_counts (feed_url, cnt)
171+ SELECT feed_url, COUNT(DISTINCT word) FROM _feed_words GROUP BY feed_url
172+ `); err != nil {
173+ return err
174+ }
181175
182- if _, err := tx.ExecContext(ctx, descUpdate); err != nil {
176+ if _, err := tx.ExecContext(ctx, `
177+ CREATE TEMP TABLE IF NOT EXISTS _word_overlap (feed_a TEXT, feed_b TEXT, common INT)
178+ `); err != nil {
179+ return err
180+ }
181+ if _, err := tx.ExecContext(ctx, `DELETE FROM _word_overlap`); err != nil {
182+ return err
183+ }
184+ if _, err := tx.ExecContext(ctx, `
185+ INSERT INTO _word_overlap (feed_a, feed_b, common)
186+ SELECT w1.feed_url, w2.feed_url, COUNT(DISTINCT w1.word)
187+ FROM _feed_words w1
188+ JOIN _feed_words w2 ON w1.word = w2.word AND w1.feed_url < w2.feed_url
189+ GROUP BY w1.feed_url, w2.feed_url
190+ HAVING COUNT(DISTINCT w1.word) > 1
191+ `); err != nil {
183192 return err
184193 }
185194
186- descInsert := fmt.Sprintf(`
195+ descInsert := `
187196 INSERT OR IGNORE INTO feed_similarity (feed_a, feed_b, jaccard)
188- SELECT feed_a, feed_b, 0 FROM (
189- SELECT w1.feed_url AS feed_a, w2.feed_url AS feed_b, COUNT(DISTINCT w1.word) AS common
190- FROM _feed_words w1
191- JOIN _feed_words w2 ON w1.word = w2.word AND w1.feed_url < w2.feed_url
192- GROUP BY w1.feed_url, w2.feed_url
193- HAVING common > 1
194- )
195- `)
197+ SELECT feed_a, feed_b, 0 FROM _word_overlap
198+ `
196199 if _, err := tx.ExecContext(ctx, descInsert); err != nil {
197200 return err
198201 }
199202
200- descBoost := fmt.Sprintf(`
203+ descUpdate := fmt.Sprintf(`
201204 UPDATE feed_similarity SET
202- jaccard = jaccard + %g * CAST(word_overlap.common AS REAL) / NULLIF(
203- (SELECT COUNT(DISTINCT word) FROM _feed_words WHERE feed_url = feed_similarity.feed_a) +
204- (SELECT COUNT(DISTINCT word) FROM _feed_words WHERE feed_url = feed_similarity.feed_b) -
205- CAST(word_overlap.common AS REAL),
205+ jaccard = jaccard + %g * CAST(_word_overlap.common AS REAL) / NULLIF(
206+ (SELECT cnt FROM _feed_word_counts WHERE feed_url = feed_similarity.feed_a) +
207+ (SELECT cnt FROM _feed_word_counts WHERE feed_url = feed_similarity.feed_b) -
208+ CAST(_word_overlap.common AS REAL),
206209 0
207210 )
208- FROM (
209- SELECT w1.feed_url AS feed_a, w2.feed_url AS feed_b, COUNT(DISTINCT w1.word) AS common
210- FROM _feed_words w1
211- JOIN _feed_words w2 ON w1.word = w2.word AND w1.feed_url < w2.feed_url
212- GROUP BY w1.feed_url, w2.feed_url
213- HAVING common > 1
214- ) word_overlap
215- WHERE feed_similarity.feed_a = word_overlap.feed_a
216- AND feed_similarity.feed_b = word_overlap.feed_b
217- AND feed_similarity.jaccard = 0
211+ FROM _word_overlap
212+ WHERE feed_similarity.feed_a = _word_overlap.feed_a
213+ AND feed_similarity.feed_b = _word_overlap.feed_b
218214 `, e.config.DescriptionWeight)
219215
220- _, err = tx.ExecContext(ctx, descBoost)
221- return err
216+ if _, err := tx.ExecContext(ctx, descUpdate); err != nil {
217+ return err
218+ }
219+
220+ return nil
222221 }
223222
224223 func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
@@ -246,31 +245,57 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
246245 FROM subscriptions s1
247246 JOIN subscriptions s2 ON s1.feed_url = s2.feed_url AND s1.user_did < s2.user_did
248247 GROUP BY s1.user_did, s2.user_did
249- HAVING COUNT(*) > 0
250248 `)
251249 if err != nil {
252250 return err
253251 }
254252
253+ if _, err := tx.ExecContext(ctx, `
254+ CREATE TEMP TABLE IF NOT EXISTS _likes_count (author_did TEXT PRIMARY KEY, cnt INT)
255+ `); err != nil {
256+ return err
257+ }
258+ if _, err := tx.ExecContext(ctx, `DELETE FROM _likes_count`); err != nil {
259+ return err
260+ }
261+ if _, err := tx.ExecContext(ctx, `
262+ INSERT INTO _likes_count (author_did, cnt)
263+ SELECT author_did, COUNT(*) FROM likes GROUP BY author_did
264+ `); err != nil {
265+ return err
266+ }
267+
268+ if _, err := tx.ExecContext(ctx, `
269+ CREATE TEMP TABLE IF NOT EXISTS _likes_overlap (user_a TEXT, user_b TEXT, common INT, PRIMARY KEY(user_a, user_b))
270+ `); err != nil {
271+ return err
272+ }
273+ if _, err := tx.ExecContext(ctx, `DELETE FROM _likes_overlap`); err != nil {
274+ return err
275+ }
276+ if _, err := tx.ExecContext(ctx, `
277+ INSERT INTO _likes_overlap (user_a, user_b, common)
278+ SELECT l1.author_did, l2.author_did, COUNT(*)
279+ FROM likes l1
280+ JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
281+ AND l1.author_did < l2.author_did
282+ GROUP BY l1.author_did, l2.author_did
283+ `); err != nil {
284+ return err
285+ }
286+
255287 likesUpdate := fmt.Sprintf(`
256288 UPDATE user_similarity SET
257- jaccard = jaccard + %g * CAST(likes_overlap.common AS REAL) / NULLIF(
258- (SELECT COUNT(*) FROM likes WHERE author_did = user_similarity.user_a) +
259- (SELECT COUNT(*) FROM likes WHERE author_did = user_similarity.user_b) -
260- CAST(likes_overlap.common AS REAL),
289+ jaccard = jaccard + %g * CAST(_likes_overlap.common AS REAL) / NULLIF(
290+ (SELECT cnt FROM _likes_count WHERE author_did = user_similarity.user_a) +
291+ (SELECT cnt FROM _likes_count WHERE author_did = user_similarity.user_b) -
292+ CAST(_likes_overlap.common AS REAL),
261293 0
262294 ),
263- common_likes = likes_overlap.common
264- FROM (
265- SELECT l1.author_did AS user_a, l2.author_did AS user_b, COUNT(*) AS common
266- FROM likes l1
267- JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
268- AND l1.author_did < l2.author_did
269- GROUP BY l1.author_did, l2.author_did
270- HAVING common > 0
271- ) likes_overlap
272- WHERE user_similarity.user_a = likes_overlap.user_a
273- AND user_similarity.user_b = likes_overlap.user_b
295+ common_likes = _likes_overlap.common
296+ FROM _likes_overlap
297+ WHERE user_similarity.user_a = _likes_overlap.user_a
298+ AND user_similarity.user_b = _likes_overlap.user_b
274299 `, e.config.LikesWeight)
275300
276301 if _, err := tx.ExecContext(ctx, likesUpdate); err != nil {
@@ -279,22 +304,20 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
279304
280305 likesInsert := fmt.Sprintf(`
281306 INSERT INTO user_similarity (user_a, user_b, jaccard, common_feeds, common_likes)
282- SELECT
283- l1.author_did,
284- l2.author_did,
285- %g * CAST(COUNT(*) AS REAL) / NULLIF(
286- (SELECT COUNT(*) FROM likes WHERE author_did = l1.author_did) +
287- (SELECT COUNT(*) FROM likes WHERE author_did = l2.author_did) -
288- CAST(COUNT(*) AS REAL),
289- 0
290- ),
291- 0,
292- COUNT(*)
293- FROM likes l1
294- JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
295- AND l1.author_did < l2.author_did
296- GROUP BY l1.author_did, l2.author_did
297- HAVING COUNT(*) > 0
307+ SELECT sub.user_a, sub.user_b, sub.jaccard, 0, sub.common
308+ FROM (
309+ SELECT
310+ lo.user_a,
311+ lo.user_b,
312+ %g * CAST(lo.common AS REAL) / NULLIF(
313+ (SELECT cnt FROM _likes_count WHERE author_did = lo.user_a) +
314+ (SELECT cnt FROM _likes_count WHERE author_did = lo.user_b) -
315+ CAST(lo.common AS REAL),
316+ 0
317+ ) AS jaccard,
318+ lo.common
319+ FROM _likes_overlap lo
320+ ) sub WHERE 1
298321 ON CONFLICT(user_a, user_b) DO UPDATE SET
299322 jaccard = jaccard + excluded.jaccard,
300323 common_likes = excluded.common_likes
@@ -321,12 +344,30 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
321344 FROM user_tags t1
322345 JOIN user_tags t2 ON t1.tag = t2.tag AND t1.author_did < t2.author_did
323346 GROUP BY t1.author_did, t2.author_did
324- HAVING COUNT(DISTINCT t1.tag) > 0
325347 `)
326348 if err != nil {
327349 return err
328350 }
329351
352+ if _, err := tx.ExecContext(ctx, `
353+ CREATE TEMP TABLE IF NOT EXISTS _tag_count (author_did TEXT PRIMARY KEY, cnt INT)
354+ `); err != nil {
355+ return err
356+ }
357+ if _, err := tx.ExecContext(ctx, `DELETE FROM _tag_count`); err != nil {
358+ return err
359+ }
360+ if _, err := tx.ExecContext(ctx, `
361+ INSERT INTO _tag_count (author_did, cnt)
362+ WITH user_tags AS (
363+ SELECT author_did, TRIM(value) AS tag FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')
364+ WHERE tags IS NOT NULL AND tags != ''
365+ )
366+ SELECT author_did, COUNT(DISTINCT tag) FROM user_tags GROUP BY author_did
367+ `); err != nil {
368+ return err
369+ }
370+
330371 _, err = tx.ExecContext(ctx, `
331372 INSERT OR IGNORE INTO user_similarity (user_a, user_b, jaccard, common_feeds, common_tags)
332373 SELECT user_a, user_b, 0, 0, 0 FROM _tag_overlap
@@ -338,8 +379,8 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
338379 tagsUpdate := fmt.Sprintf(`
339380 UPDATE user_similarity SET
340381 jaccard = jaccard + %g * CAST(_tag_overlap.common AS REAL) / NULLIF(
341- (SELECT COUNT(DISTINCT TRIM(value)) FROM annotations a, json_each('["' || REPLACE(a.tags, ',', '","') || '"]') WHERE a.author_did = user_similarity.user_a AND a.tags IS NOT NULL AND a.tags != '') +
342- (SELECT COUNT(DISTINCT TRIM(value)) FROM annotations a, json_each('["' || REPLACE(a.tags, ',', '","') || '"]') WHERE a.author_did = user_similarity.user_b AND a.tags IS NOT NULL AND a.tags != '') -
382+ (SELECT cnt FROM _tag_count WHERE author_did = user_similarity.user_a) +
383+ (SELECT cnt FROM _tag_count WHERE author_did = user_similarity.user_b) -
343384 CAST(_tag_overlap.common AS REAL),
344385 0
345386 ),
@@ -401,64 +442,93 @@ func (e *Engine) ComputeUserSimilarityForUser(ctx context.Context, userDID strin
401442 JOIN subscriptions s2 ON s1.feed_url = s2.feed_url AND s2.user_did != ?
402443 WHERE s1.user_did = ?
403444 GROUP BY s2.user_did
404- HAVING COUNT(*) > 0
405445 `, userDID, userDID, userDID, userDID, userDID)
406446 if err != nil {
407447 return err
408448 }
409449
450+ if _, err := tx.ExecContext(ctx, `
451+ CREATE TEMP TABLE IF NOT EXISTS _per_user_likes_count (author_did TEXT PRIMARY KEY, cnt INT)
452+ `); err != nil {
453+ return err
454+ }
455+ if _, err := tx.ExecContext(ctx, `DELETE FROM _per_user_likes_count`); err != nil {
456+ return err
457+ }
458+ if _, err := tx.ExecContext(ctx, `
459+ INSERT INTO _per_user_likes_count (author_did, cnt)
460+ SELECT author_did, COUNT(*) FROM likes
461+ WHERE author_did = ? OR author_did IN (
462+ SELECT CASE WHEN user_a = ? THEN user_b ELSE user_a END
463+ FROM user_similarity
464+ WHERE user_a = ? OR user_b = ?
465+ )
466+ GROUP BY author_did
467+ `, userDID, userDID, userDID, userDID); err != nil {
468+ return err
469+ }
470+
471+ if _, err := tx.ExecContext(ctx, `
472+ CREATE TEMP TABLE IF NOT EXISTS _per_user_likes_overlap (peer TEXT, common INT)
473+ `); err != nil {
474+ return err
475+ }
476+ if _, err := tx.ExecContext(ctx, `DELETE FROM _per_user_likes_overlap`); err != nil {
477+ return err
478+ }
479+ if _, err := tx.ExecContext(ctx, `
480+ INSERT INTO _per_user_likes_overlap (peer, common)
481+ SELECT l2.author_did, COUNT(*)
482+ FROM likes l1
483+ JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
484+ AND l2.author_did != ?
485+ WHERE l1.author_did = ?
486+ GROUP BY l2.author_did
487+ `, userDID, userDID); err != nil {
488+ return err
489+ }
490+
410491 likesPerUser := fmt.Sprintf(`
411492 UPDATE user_similarity SET
412- jaccard = jaccard + %g * CAST(likes_overlap.common AS REAL) / NULLIF(
413- (SELECT COUNT(*) FROM likes WHERE author_did = ?) +
414- (SELECT COUNT(*) FROM likes WHERE author_did =
493+ jaccard = jaccard + %g * CAST(_per_user_likes_overlap.common AS REAL) / NULLIF(
494+ (SELECT cnt FROM _per_user_likes_count WHERE author_did = ?) +
495+ (SELECT cnt FROM _per_user_likes_count WHERE author_did =
415496 CASE WHEN user_similarity.user_a = ? THEN user_similarity.user_b ELSE user_similarity.user_a END
416- ) - CAST(likes_overlap.common AS REAL),
497+ ) - CAST(_per_user_likes_overlap.common AS REAL),
417498 0
418499 ),
419- common_likes = likes_overlap.common
420- FROM (
421- SELECT l2.author_did AS peer, COUNT(*) AS common
422- FROM likes l1
423- JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
424- AND l2.author_did != ?
425- WHERE l1.author_did = ?
426- GROUP BY l2.author_did
427- HAVING common > 0
428- ) likes_overlap
429- WHERE user_similarity.user_a = likes_overlap.peer
430- OR user_similarity.user_b = likes_overlap.peer
500+ common_likes = _per_user_likes_overlap.common
501+ FROM _per_user_likes_overlap
502+ WHERE user_similarity.user_a = _per_user_likes_overlap.peer
503+ OR user_similarity.user_b = _per_user_likes_overlap.peer
431504 `, e.config.LikesWeight)
432505
433- if _, err := tx.ExecContext(ctx, likesPerUser, userDID, userDID, userDID, userDID); err != nil {
506+ if _, err := tx.ExecContext(ctx, likesPerUser, userDID, userDID); err != nil {
434507 return err
435508 }
436509
437510 likesInsert := fmt.Sprintf(`
438511 INSERT INTO user_similarity (user_a, user_b, jaccard, common_feeds, common_likes)
439- SELECT
440- MIN(?, l2.author_did),
441- MAX(?, l2.author_did),
442- %g * CAST(COUNT(*) AS REAL) / NULLIF(
443- (SELECT COUNT(*) FROM likes WHERE author_did = ?) +
444- (SELECT COUNT(*) FROM likes WHERE author_did = l2.author_did) -
445- CAST(COUNT(*) AS REAL),
446- 0
447- ),
448- 0,
449- COUNT(*)
450- FROM likes l1
451- JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
452- AND l2.author_did != ?
453- WHERE l1.author_did = ?
454- GROUP BY l2.author_did
455- HAVING COUNT(*) > 0
512+ SELECT sub.user_a, sub.user_b, sub.jaccard, 0, sub.common
513+ FROM (
514+ SELECT
515+ MIN(?, lo.peer) AS user_a,
516+ MAX(?, lo.peer) AS user_b,
517+ %g * CAST(lo.common AS REAL) / NULLIF(
518+ (SELECT cnt FROM _per_user_likes_count WHERE author_did = ?) +
519+ (SELECT cnt FROM _per_user_likes_count WHERE author_did = lo.peer) -
520+ CAST(lo.common AS REAL),
521+ 0
522+ ) AS jaccard,
523+ lo.common
524+ FROM _per_user_likes_overlap lo
525+ ) sub WHERE 1
456526 ON CONFLICT(user_a, user_b) DO UPDATE SET
457527 jaccard = jaccard + excluded.jaccard,
458528 common_likes = excluded.common_likes
459529 `, e.config.LikesWeight)
460530
461- if _, err := tx.ExecContext(ctx, likesInsert, userDID, userDID, userDID, userDID, userDID); err != nil {
531+ if _, err := tx.ExecContext(ctx, likesInsert, userDID, userDID, userDID); err != nil {
462532 return err
463533 }
464534
@@ -480,7 +550,6 @@ func (e *Engine) ComputeUserSimilarityForUser(ctx context.Context, userDID strin
480550 JOIN user_tags t2 ON t1.tag = t2.tag AND t2.author_did != ?
481551 WHERE t1.author_did = ?
482552 GROUP BY t2.author_did
483- HAVING COUNT(DISTINCT t1.tag) > 0
484553 `, userDID, userDID)
485554 if err != nil {
486555 return err
@@ -494,14 +563,34 @@ func (e *Engine) ComputeUserSimilarityForUser(ctx context.Context, userDID strin
494563 return err
495564 }
496565
566+ if _, err := tx.ExecContext(ctx, `
567+ CREATE TEMP TABLE IF NOT EXISTS _per_user_tag_count (author_did TEXT PRIMARY KEY, cnt INT)
568+ `); err != nil {
569+ return err
570+ }
571+ if _, err := tx.ExecContext(ctx, `DELETE FROM _per_user_tag_count`); err != nil {
572+ return err
573+ }
574+ if _, err := tx.ExecContext(ctx, `
575+ INSERT INTO _per_user_tag_count (author_did, cnt)
576+ WITH user_tags AS (
577+ SELECT author_did, TRIM(value) AS tag FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')
578+ WHERE tags IS NOT NULL AND tags != ''
579+ )
580+ SELECT author_did, COUNT(DISTINCT tag) FROM user_tags
581+ WHERE author_did = ? OR author_did IN (SELECT peer FROM _user_tag_overlap)
582+ GROUP BY author_did
583+ `, userDID); err != nil {
584+ return err
585+ }
586+
497587 tagsPerUser := fmt.Sprintf(`
498588 UPDATE user_similarity SET
499589 jaccard = jaccard + %g * CAST(_user_tag_overlap.common AS REAL) / NULLIF(
500- (SELECT COUNT(DISTINCT TRIM(value)) FROM annotations a, json_each('["' || REPLACE(a.tags, ',', '","') || '"]') WHERE a.author_did = ? AND a.tags IS NOT NULL AND a.tags != '') +
501- (SELECT COUNT(DISTINCT TRIM(value)) FROM annotations a, json_each('["' || REPLACE(a.tags, ',', '","') || '"]') WHERE a.author_did =
590+ (SELECT cnt FROM _per_user_tag_count WHERE author_did = ?) +
591+ (SELECT cnt FROM _per_user_tag_count WHERE author_did =
502592 CASE WHEN user_similarity.user_a = ? THEN user_similarity.user_b ELSE user_similarity.user_a END
503- AND a.tags IS NOT NULL AND a.tags != '') -
504- CAST(_user_tag_overlap.common AS REAL),
593+ ) - CAST(_user_tag_overlap.common AS REAL),
505594 0
506595 ),
507596 common_tags = _user_tag_overlap.common
@@ -607,7 +696,6 @@ func (e *Engine) computeArticleRecommendationsForUser(ctx context.Context, userD
607696 SELECT 1 FROM likes ul WHERE ul.author_did = ? AND ul.feed_url = l.feed_url AND ul.article_url = l.article_url
608697 )
609698 GROUP BY l.feed_url, l.article_url
610- HAVING COUNT(*) > 0
611699 ORDER BY score DESC
612700 `, e.config.SimilarityThreshold, e.config.SimilarityThreshold)
613701
@@ -68,7 +68,6 @@ func (e *Engine) ComputeArticleRecommendations(ctx context.Context) error {
68 SELECT 1 FROM likes ul WHERE ul.author_did = targets.target AND ul.feed_url = l.feed_url AND ul.article_url = l.article_url68 SELECT 1 FROM likes ul WHERE ul.author_did = targets.target AND ul.feed_url = l.feed_url AND ul.article_url = l.article_url
69 )69 )
70 GROUP BY targets.target, l.feed_url, l.article_url70 GROUP BY targets.target, l.feed_url, l.article_url
71- HAVING COUNT(*) > 0
72 ORDER BY score DESC71 ORDER BY score DESC
73 `, e.config.SimilarityThreshold, e.config.SimilarityThreshold)72 `, e.config.SimilarityThreshold, e.config.SimilarityThreshold)
74 73
@@ -117,7 +116,6 @@ func (e *Engine) ComputeFeedSimilarity(ctx context.Context) error {
117 JOIN feeds f1 ON f1.feed_url = s1.feed_url116 JOIN feeds f1 ON f1.feed_url = s1.feed_url
118 JOIN feeds f2 ON f2.feed_url = s2.feed_url117 JOIN feeds f2 ON f2.feed_url = s2.feed_url
119 GROUP BY s1.feed_url, s2.feed_url118 GROUP BY s1.feed_url, s2.feed_url
120- HAVING COUNT(*) > 0
121 `)119 `)
122 if err != nil {120 if err != nil {
123 return err121 return err
@@ -160,65 +158,66 @@ func (e *Engine) computeDescriptionSimilarity(ctx context.Context, tx *sql.Tx) e
160 return err158 return err
161 }159 }
162 160
163- descUpdate := fmt.Sprintf(`161+ if _, err := tx.ExecContext(ctx, `
164- UPDATE feed_similarity SET162+ CREATE TEMP TABLE IF NOT EXISTS _feed_word_counts (feed_url TEXT PRIMARY KEY, cnt INT)
165- jaccard = jaccard + %g * CAST(word_overlap.common AS REAL) / NULLIF(163+ `); err != nil {
166- (SELECT COUNT(DISTINCT word) FROM _feed_words WHERE feed_url = feed_similarity.feed_a) +164+ return err
167- (SELECT COUNT(DISTINCT word) FROM _feed_words WHERE feed_url = feed_similarity.feed_b) -165+ }
168- CAST(word_overlap.common AS REAL),166+ if _, err := tx.ExecContext(ctx, `DELETE FROM _feed_word_counts`); err != nil {
169- 0167+ return err
170- )168+ }
171- FROM (169+ if _, err := tx.ExecContext(ctx, `
172- SELECT w1.feed_url AS feed_a, w2.feed_url AS feed_b, COUNT(DISTINCT w1.word) AS common170+ INSERT INTO _feed_word_counts (feed_url, cnt)
173- FROM _feed_words w1171+ SELECT feed_url, COUNT(DISTINCT word) FROM _feed_words GROUP BY feed_url
174- JOIN _feed_words w2 ON w1.word = w2.word AND w1.feed_url < w2.feed_url172+ `); err != nil {
175- GROUP BY w1.feed_url, w2.feed_url173+ return err
176- HAVING common > 1174+ }
177- ) word_overlap
178- WHERE feed_similarity.feed_a = word_overlap.feed_a
179- AND feed_similarity.feed_b = word_overlap.feed_b
180- `, e.config.DescriptionWeight)
181 175
182- if _, err := tx.ExecContext(ctx, descUpdate); err != nil {176+ if _, err := tx.ExecContext(ctx, `
177+ CREATE TEMP TABLE IF NOT EXISTS _word_overlap (feed_a TEXT, feed_b TEXT, common INT)
178+ `); err != nil {
179+ return err
180+ }
181+ if _, err := tx.ExecContext(ctx, `DELETE FROM _word_overlap`); err != nil {
182+ return err
183+ }
184+ if _, err := tx.ExecContext(ctx, `
185+ INSERT INTO _word_overlap (feed_a, feed_b, common)
186+ SELECT w1.feed_url, w2.feed_url, COUNT(DISTINCT w1.word)
187+ FROM _feed_words w1
188+ JOIN _feed_words w2 ON w1.word = w2.word AND w1.feed_url < w2.feed_url
189+ GROUP BY w1.feed_url, w2.feed_url
190+ HAVING COUNT(DISTINCT w1.word) > 1
191+ `); err != nil {
183 return err192 return err
184 }193 }
185 194
186- descInsert := fmt.Sprintf(`195+ descInsert := `
187 INSERT OR IGNORE INTO feed_similarity (feed_a, feed_b, jaccard)196 INSERT OR IGNORE INTO feed_similarity (feed_a, feed_b, jaccard)
188- SELECT feed_a, feed_b, 0 FROM (197+ SELECT feed_a, feed_b, 0 FROM _word_overlap
189- SELECT w1.feed_url AS feed_a, w2.feed_url AS feed_b, COUNT(DISTINCT w1.word) AS common198+ `
190- FROM _feed_words w1
191- JOIN _feed_words w2 ON w1.word = w2.word AND w1.feed_url < w2.feed_url
192- GROUP BY w1.feed_url, w2.feed_url
193- HAVING common > 1
194- )
195- `)
196 if _, err := tx.ExecContext(ctx, descInsert); err != nil {199 if _, err := tx.ExecContext(ctx, descInsert); err != nil {
197 return err200 return err
198 }201 }
199 202
200- descBoost := fmt.Sprintf(`203+ descUpdate := fmt.Sprintf(`
201 UPDATE feed_similarity SET204 UPDATE feed_similarity SET
202- jaccard = jaccard + %g * CAST(word_overlap.common AS REAL) / NULLIF(205+ jaccard = jaccard + %g * CAST(_word_overlap.common AS REAL) / NULLIF(
203- (SELECT COUNT(DISTINCT word) FROM _feed_words WHERE feed_url = feed_similarity.feed_a) +206+ (SELECT cnt FROM _feed_word_counts WHERE feed_url = feed_similarity.feed_a) +
204- (SELECT COUNT(DISTINCT word) FROM _feed_words WHERE feed_url = feed_similarity.feed_b) -207+ (SELECT cnt FROM _feed_word_counts WHERE feed_url = feed_similarity.feed_b) -
205- CAST(word_overlap.common AS REAL),208+ CAST(_word_overlap.common AS REAL),
206 0209 0
207 )210 )
208- FROM (211+ FROM _word_overlap
209- SELECT w1.feed_url AS feed_a, w2.feed_url AS feed_b, COUNT(DISTINCT w1.word) AS common212+ WHERE feed_similarity.feed_a = _word_overlap.feed_a
210- FROM _feed_words w1213+ AND feed_similarity.feed_b = _word_overlap.feed_b
211- JOIN _feed_words w2 ON w1.word = w2.word AND w1.feed_url < w2.feed_url
212- GROUP BY w1.feed_url, w2.feed_url
213- HAVING common > 1
214- ) word_overlap
215- WHERE feed_similarity.feed_a = word_overlap.feed_a
216- AND feed_similarity.feed_b = word_overlap.feed_b
217- AND feed_similarity.jaccard = 0
218 `, e.config.DescriptionWeight)214 `, e.config.DescriptionWeight)
219 215
220- _, err = tx.ExecContext(ctx, descBoost)216+ if _, err := tx.ExecContext(ctx, descUpdate); err != nil {
221- return err217+ return err
218+ }
219+
220+ return nil
222 }221 }
223 222
224 func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {223 func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
@@ -246,31 +245,57 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
246 FROM subscriptions s1245 FROM subscriptions s1
247 JOIN subscriptions s2 ON s1.feed_url = s2.feed_url AND s1.user_did < s2.user_did246 JOIN subscriptions s2 ON s1.feed_url = s2.feed_url AND s1.user_did < s2.user_did
248 GROUP BY s1.user_did, s2.user_did247 GROUP BY s1.user_did, s2.user_did
249- HAVING COUNT(*) > 0
250 `)248 `)
251 if err != nil {249 if err != nil {
252 return err250 return err
253 }251 }
254 252
253+ if _, err := tx.ExecContext(ctx, `
254+ CREATE TEMP TABLE IF NOT EXISTS _likes_count (author_did TEXT PRIMARY KEY, cnt INT)
255+ `); err != nil {
256+ return err
257+ }
258+ if _, err := tx.ExecContext(ctx, `DELETE FROM _likes_count`); err != nil {
259+ return err
260+ }
261+ if _, err := tx.ExecContext(ctx, `
262+ INSERT INTO _likes_count (author_did, cnt)
263+ SELECT author_did, COUNT(*) FROM likes GROUP BY author_did
264+ `); err != nil {
265+ return err
266+ }
267+
268+ if _, err := tx.ExecContext(ctx, `
269+ CREATE TEMP TABLE IF NOT EXISTS _likes_overlap (user_a TEXT, user_b TEXT, common INT, PRIMARY KEY(user_a, user_b))
270+ `); err != nil {
271+ return err
272+ }
273+ if _, err := tx.ExecContext(ctx, `DELETE FROM _likes_overlap`); err != nil {
274+ return err
275+ }
276+ if _, err := tx.ExecContext(ctx, `
277+ INSERT INTO _likes_overlap (user_a, user_b, common)
278+ SELECT l1.author_did, l2.author_did, COUNT(*)
279+ FROM likes l1
280+ JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
281+ AND l1.author_did < l2.author_did
282+ GROUP BY l1.author_did, l2.author_did
283+ `); err != nil {
284+ return err
285+ }
286+
255 likesUpdate := fmt.Sprintf(`287 likesUpdate := fmt.Sprintf(`
256 UPDATE user_similarity SET288 UPDATE user_similarity SET
257- jaccard = jaccard + %g * CAST(likes_overlap.common AS REAL) / NULLIF(289+ jaccard = jaccard + %g * CAST(_likes_overlap.common AS REAL) / NULLIF(
258- (SELECT COUNT(*) FROM likes WHERE author_did = user_similarity.user_a) +290+ (SELECT cnt FROM _likes_count WHERE author_did = user_similarity.user_a) +
259- (SELECT COUNT(*) FROM likes WHERE author_did = user_similarity.user_b) -291+ (SELECT cnt FROM _likes_count WHERE author_did = user_similarity.user_b) -
260- CAST(likes_overlap.common AS REAL),292+ CAST(_likes_overlap.common AS REAL),
261 0293 0
262 ),294 ),
263- common_likes = likes_overlap.common295+ common_likes = _likes_overlap.common
264- FROM (296+ FROM _likes_overlap
265- SELECT l1.author_did AS user_a, l2.author_did AS user_b, COUNT(*) AS common297+ WHERE user_similarity.user_a = _likes_overlap.user_a
266- FROM likes l1298+ AND user_similarity.user_b = _likes_overlap.user_b
267- JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
268- AND l1.author_did < l2.author_did
269- GROUP BY l1.author_did, l2.author_did
270- HAVING common > 0
271- ) likes_overlap
272- WHERE user_similarity.user_a = likes_overlap.user_a
273- AND user_similarity.user_b = likes_overlap.user_b
274 `, e.config.LikesWeight)299 `, e.config.LikesWeight)
275 300
276 if _, err := tx.ExecContext(ctx, likesUpdate); err != nil {301 if _, err := tx.ExecContext(ctx, likesUpdate); err != nil {
@@ -279,22 +304,20 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
279 304
280 likesInsert := fmt.Sprintf(`305 likesInsert := fmt.Sprintf(`
281 INSERT INTO user_similarity (user_a, user_b, jaccard, common_feeds, common_likes)306 INSERT INTO user_similarity (user_a, user_b, jaccard, common_feeds, common_likes)
282- SELECT307+ SELECT sub.user_a, sub.user_b, sub.jaccard, 0, sub.common
283- l1.author_did,308+ FROM (
284- l2.author_did,309+ SELECT
285- %g * CAST(COUNT(*) AS REAL) / NULLIF(310+ lo.user_a,
286- (SELECT COUNT(*) FROM likes WHERE author_did = l1.author_did) +311+ lo.user_b,
287- (SELECT COUNT(*) FROM likes WHERE author_did = l2.author_did) -312+ %g * CAST(lo.common AS REAL) / NULLIF(
288- CAST(COUNT(*) AS REAL),313+ (SELECT cnt FROM _likes_count WHERE author_did = lo.user_a) +
289- 0314+ (SELECT cnt FROM _likes_count WHERE author_did = lo.user_b) -
290- ),315+ CAST(lo.common AS REAL),
291- 0,316+ 0
292- COUNT(*)317+ ) AS jaccard,
293- FROM likes l1318+ lo.common
294- JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url319+ FROM _likes_overlap lo
295- AND l1.author_did < l2.author_did320+ ) sub WHERE 1
296- GROUP BY l1.author_did, l2.author_did
297- HAVING COUNT(*) > 0
298 ON CONFLICT(user_a, user_b) DO UPDATE SET321 ON CONFLICT(user_a, user_b) DO UPDATE SET
299 jaccard = jaccard + excluded.jaccard,322 jaccard = jaccard + excluded.jaccard,
300 common_likes = excluded.common_likes323 common_likes = excluded.common_likes
@@ -321,12 +344,30 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
321 FROM user_tags t1344 FROM user_tags t1
322 JOIN user_tags t2 ON t1.tag = t2.tag AND t1.author_did < t2.author_did345 JOIN user_tags t2 ON t1.tag = t2.tag AND t1.author_did < t2.author_did
323 GROUP BY t1.author_did, t2.author_did346 GROUP BY t1.author_did, t2.author_did
324- HAVING COUNT(DISTINCT t1.tag) > 0
325 `)347 `)
326 if err != nil {348 if err != nil {
327 return err349 return err
328 }350 }
329 351
352+ if _, err := tx.ExecContext(ctx, `
353+ CREATE TEMP TABLE IF NOT EXISTS _tag_count (author_did TEXT PRIMARY KEY, cnt INT)
354+ `); err != nil {
355+ return err
356+ }
357+ if _, err := tx.ExecContext(ctx, `DELETE FROM _tag_count`); err != nil {
358+ return err
359+ }
360+ if _, err := tx.ExecContext(ctx, `
361+ INSERT INTO _tag_count (author_did, cnt)
362+ WITH user_tags AS (
363+ SELECT author_did, TRIM(value) AS tag FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')
364+ WHERE tags IS NOT NULL AND tags != ''
365+ )
366+ SELECT author_did, COUNT(DISTINCT tag) FROM user_tags GROUP BY author_did
367+ `); err != nil {
368+ return err
369+ }
370+
330 _, err = tx.ExecContext(ctx, `371 _, err = tx.ExecContext(ctx, `
331 INSERT OR IGNORE INTO user_similarity (user_a, user_b, jaccard, common_feeds, common_tags)372 INSERT OR IGNORE INTO user_similarity (user_a, user_b, jaccard, common_feeds, common_tags)
332 SELECT user_a, user_b, 0, 0, 0 FROM _tag_overlap373 SELECT user_a, user_b, 0, 0, 0 FROM _tag_overlap
@@ -338,8 +379,8 @@ func (e *Engine) ComputeUserSimilarity(ctx context.Context) error {
338 tagsUpdate := fmt.Sprintf(`379 tagsUpdate := fmt.Sprintf(`
339 UPDATE user_similarity SET380 UPDATE user_similarity SET
340 jaccard = jaccard + %g * CAST(_tag_overlap.common AS REAL) / NULLIF(381 jaccard = jaccard + %g * CAST(_tag_overlap.common AS REAL) / NULLIF(
341- (SELECT COUNT(DISTINCT TRIM(value)) FROM annotations a, json_each('["' || REPLACE(a.tags, ',', '","') || '"]') WHERE a.author_did = user_similarity.user_a AND a.tags IS NOT NULL AND a.tags != '') +382+ (SELECT cnt FROM _tag_count WHERE author_did = user_similarity.user_a) +
342- (SELECT COUNT(DISTINCT TRIM(value)) FROM annotations a, json_each('["' || REPLACE(a.tags, ',', '","') || '"]') WHERE a.author_did = user_similarity.user_b AND a.tags IS NOT NULL AND a.tags != '') -383+ (SELECT cnt FROM _tag_count WHERE author_did = user_similarity.user_b) -
343 CAST(_tag_overlap.common AS REAL),384 CAST(_tag_overlap.common AS REAL),
344 0385 0
345 ),386 ),
@@ -401,64 +442,93 @@ func (e *Engine) ComputeUserSimilarityForUser(ctx context.Context, userDID strin
401 JOIN subscriptions s2 ON s1.feed_url = s2.feed_url AND s2.user_did != ?442 JOIN subscriptions s2 ON s1.feed_url = s2.feed_url AND s2.user_did != ?
402 WHERE s1.user_did = ?443 WHERE s1.user_did = ?
403 GROUP BY s2.user_did444 GROUP BY s2.user_did
404- HAVING COUNT(*) > 0
405 `, userDID, userDID, userDID, userDID, userDID)445 `, userDID, userDID, userDID, userDID, userDID)
406 if err != nil {446 if err != nil {
407 return err447 return err
408 }448 }
409 449
450+ if _, err := tx.ExecContext(ctx, `
451+ CREATE TEMP TABLE IF NOT EXISTS _per_user_likes_count (author_did TEXT PRIMARY KEY, cnt INT)
452+ `); err != nil {
453+ return err
454+ }
455+ if _, err := tx.ExecContext(ctx, `DELETE FROM _per_user_likes_count`); err != nil {
456+ return err
457+ }
458+ if _, err := tx.ExecContext(ctx, `
459+ INSERT INTO _per_user_likes_count (author_did, cnt)
460+ SELECT author_did, COUNT(*) FROM likes
461+ WHERE author_did = ? OR author_did IN (
462+ SELECT CASE WHEN user_a = ? THEN user_b ELSE user_a END
463+ FROM user_similarity
464+ WHERE user_a = ? OR user_b = ?
465+ )
466+ GROUP BY author_did
467+ `, userDID, userDID, userDID, userDID); err != nil {
468+ return err
469+ }
470+
471+ if _, err := tx.ExecContext(ctx, `
472+ CREATE TEMP TABLE IF NOT EXISTS _per_user_likes_overlap (peer TEXT, common INT)
473+ `); err != nil {
474+ return err
475+ }
476+ if _, err := tx.ExecContext(ctx, `DELETE FROM _per_user_likes_overlap`); err != nil {
477+ return err
478+ }
479+ if _, err := tx.ExecContext(ctx, `
480+ INSERT INTO _per_user_likes_overlap (peer, common)
481+ SELECT l2.author_did, COUNT(*)
482+ FROM likes l1
483+ JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
484+ AND l2.author_did != ?
485+ WHERE l1.author_did = ?
486+ GROUP BY l2.author_did
487+ `, userDID, userDID); err != nil {
488+ return err
489+ }
490+
410 likesPerUser := fmt.Sprintf(`491 likesPerUser := fmt.Sprintf(`
411 UPDATE user_similarity SET492 UPDATE user_similarity SET
412- jaccard = jaccard + %g * CAST(likes_overlap.common AS REAL) / NULLIF(493+ jaccard = jaccard + %g * CAST(_per_user_likes_overlap.common AS REAL) / NULLIF(
413- (SELECT COUNT(*) FROM likes WHERE author_did = ?) +494+ (SELECT cnt FROM _per_user_likes_count WHERE author_did = ?) +
414- (SELECT COUNT(*) FROM likes WHERE author_did =495+ (SELECT cnt FROM _per_user_likes_count WHERE author_did =
415 CASE WHEN user_similarity.user_a = ? THEN user_similarity.user_b ELSE user_similarity.user_a END496 CASE WHEN user_similarity.user_a = ? THEN user_similarity.user_b ELSE user_similarity.user_a END
416- ) - CAST(likes_overlap.common AS REAL),497+ ) - CAST(_per_user_likes_overlap.common AS REAL),
417 0498 0
418 ),499 ),
419- common_likes = likes_overlap.common500+ common_likes = _per_user_likes_overlap.common
420- FROM (501+ FROM _per_user_likes_overlap
421- SELECT l2.author_did AS peer, COUNT(*) AS common502+ WHERE user_similarity.user_a = _per_user_likes_overlap.peer
422- FROM likes l1503+ OR user_similarity.user_b = _per_user_likes_overlap.peer
423- JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url
424- AND l2.author_did != ?
425- WHERE l1.author_did = ?
426- GROUP BY l2.author_did
427- HAVING common > 0
428- ) likes_overlap
429- WHERE user_similarity.user_a = likes_overlap.peer
430- OR user_similarity.user_b = likes_overlap.peer
431 `, e.config.LikesWeight)504 `, e.config.LikesWeight)
432 505
433- if _, err := tx.ExecContext(ctx, likesPerUser, userDID, userDID, userDID, userDID); err != nil {506+ if _, err := tx.ExecContext(ctx, likesPerUser, userDID, userDID); err != nil {
434 return err507 return err
435 }508 }
436 509
437 likesInsert := fmt.Sprintf(`510 likesInsert := fmt.Sprintf(`
438 INSERT INTO user_similarity (user_a, user_b, jaccard, common_feeds, common_likes)511 INSERT INTO user_similarity (user_a, user_b, jaccard, common_feeds, common_likes)
439- SELECT512+ SELECT sub.user_a, sub.user_b, sub.jaccard, 0, sub.common
440- MIN(?, l2.author_did),513+ FROM (
441- MAX(?, l2.author_did),514+ SELECT
442- %g * CAST(COUNT(*) AS REAL) / NULLIF(515+ MIN(?, lo.peer) AS user_a,
443- (SELECT COUNT(*) FROM likes WHERE author_did = ?) +516+ MAX(?, lo.peer) AS user_b,
444- (SELECT COUNT(*) FROM likes WHERE author_did = l2.author_did) -517+ %g * CAST(lo.common AS REAL) / NULLIF(
445- CAST(COUNT(*) AS REAL),518+ (SELECT cnt FROM _per_user_likes_count WHERE author_did = ?) +
446- 0519+ (SELECT cnt FROM _per_user_likes_count WHERE author_did = lo.peer) -
447- ),520+ CAST(lo.common AS REAL),
448- 0,521+ 0
449- COUNT(*)522+ ) AS jaccard,
450- FROM likes l1523+ lo.common
451- JOIN likes l2 ON l1.feed_url = l2.feed_url AND l1.article_url = l2.article_url524+ FROM _per_user_likes_overlap lo
452- AND l2.author_did != ?525+ ) sub WHERE 1
453- WHERE l1.author_did = ?
454- GROUP BY l2.author_did
455- HAVING COUNT(*) > 0
456 ON CONFLICT(user_a, user_b) DO UPDATE SET526 ON CONFLICT(user_a, user_b) DO UPDATE SET
457 jaccard = jaccard + excluded.jaccard,527 jaccard = jaccard + excluded.jaccard,
458 common_likes = excluded.common_likes528 common_likes = excluded.common_likes
459 `, e.config.LikesWeight)529 `, e.config.LikesWeight)
460 530
461- if _, err := tx.ExecContext(ctx, likesInsert, userDID, userDID, userDID, userDID, userDID); err != nil {531+ if _, err := tx.ExecContext(ctx, likesInsert, userDID, userDID, userDID); err != nil {
462 return err532 return err
463 }533 }
464 534
@@ -480,7 +550,6 @@ func (e *Engine) ComputeUserSimilarityForUser(ctx context.Context, userDID strin
480 JOIN user_tags t2 ON t1.tag = t2.tag AND t2.author_did != ?550 JOIN user_tags t2 ON t1.tag = t2.tag AND t2.author_did != ?
481 WHERE t1.author_did = ?551 WHERE t1.author_did = ?
482 GROUP BY t2.author_did552 GROUP BY t2.author_did
483- HAVING COUNT(DISTINCT t1.tag) > 0
484 `, userDID, userDID)553 `, userDID, userDID)
485 if err != nil {554 if err != nil {
486 return err555 return err
@@ -494,14 +563,34 @@ func (e *Engine) ComputeUserSimilarityForUser(ctx context.Context, userDID strin
494 return err563 return err
495 }564 }
496 565
566+ if _, err := tx.ExecContext(ctx, `
567+ CREATE TEMP TABLE IF NOT EXISTS _per_user_tag_count (author_did TEXT PRIMARY KEY, cnt INT)
568+ `); err != nil {
569+ return err
570+ }
571+ if _, err := tx.ExecContext(ctx, `DELETE FROM _per_user_tag_count`); err != nil {
572+ return err
573+ }
574+ if _, err := tx.ExecContext(ctx, `
575+ INSERT INTO _per_user_tag_count (author_did, cnt)
576+ WITH user_tags AS (
577+ SELECT author_did, TRIM(value) AS tag FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')
578+ WHERE tags IS NOT NULL AND tags != ''
579+ )
580+ SELECT author_did, COUNT(DISTINCT tag) FROM user_tags
581+ WHERE author_did = ? OR author_did IN (SELECT peer FROM _user_tag_overlap)
582+ GROUP BY author_did
583+ `, userDID); err != nil {
584+ return err
585+ }
586+
497 tagsPerUser := fmt.Sprintf(`587 tagsPerUser := fmt.Sprintf(`
498 UPDATE user_similarity SET588 UPDATE user_similarity SET
499 jaccard = jaccard + %g * CAST(_user_tag_overlap.common AS REAL) / NULLIF(589 jaccard = jaccard + %g * CAST(_user_tag_overlap.common AS REAL) / NULLIF(
500- (SELECT COUNT(DISTINCT TRIM(value)) FROM annotations a, json_each('["' || REPLACE(a.tags, ',', '","') || '"]') WHERE a.author_did = ? AND a.tags IS NOT NULL AND a.tags != '') +590+ (SELECT cnt FROM _per_user_tag_count WHERE author_did = ?) +
501- (SELECT COUNT(DISTINCT TRIM(value)) FROM annotations a, json_each('["' || REPLACE(a.tags, ',', '","') || '"]') WHERE a.author_did =591+ (SELECT cnt FROM _per_user_tag_count WHERE author_did =
502 CASE WHEN user_similarity.user_a = ? THEN user_similarity.user_b ELSE user_similarity.user_a END592 CASE WHEN user_similarity.user_a = ? THEN user_similarity.user_b ELSE user_similarity.user_a END
503- AND a.tags IS NOT NULL AND a.tags != '') -593+ ) - CAST(_user_tag_overlap.common AS REAL),
504- CAST(_user_tag_overlap.common AS REAL),
505 0594 0
506 ),595 ),
507 common_tags = _user_tag_overlap.common596 common_tags = _user_tag_overlap.common
@@ -607,7 +696,6 @@ func (e *Engine) computeArticleRecommendationsForUser(ctx context.Context, userD
607 SELECT 1 FROM likes ul WHERE ul.author_did = ? AND ul.feed_url = l.feed_url AND ul.article_url = l.article_url696 SELECT 1 FROM likes ul WHERE ul.author_did = ? AND ul.feed_url = l.feed_url AND ul.article_url = l.article_url
608 )697 )
609 GROUP BY l.feed_url, l.article_url698 GROUP BY l.feed_url, l.article_url
610- HAVING COUNT(*) > 0
611 ORDER BY score DESC699 ORDER BY score DESC
612 `, e.config.SimilarityThreshold, e.config.SimilarityThreshold)700 `, e.config.SimilarityThreshold, e.config.SimilarityThreshold)
613 701
modified internal/db/db.go +1 -0
@@ -207,6 +207,7 @@ var schema = []string{
207207 `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`,
208208 `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
209209 `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
210+ `CREATE INDEX IF NOT EXISTS idx_user_similarity_b ON user_similarity(user_b)`,
210211 `CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle)`,
211212 `CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,
212213 `CREATE TRIGGER IF NOT EXISTS articles_ai AFTER INSERT ON articles BEGIN
@@ -207,6 +207,7 @@ var schema = []string{
207 `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`,207 `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`,
208 `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,208 `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
209 `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,209 `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
210+ `CREATE INDEX IF NOT EXISTS idx_user_similarity_b ON user_similarity(user_b)`,
210 `CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle)`,211 `CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle)`,
211 `CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,212 `CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,
212 `CREATE TRIGGER IF NOT EXISTS articles_ai AFTER INSERT ON articles BEGIN213 `CREATE TRIGGER IF NOT EXISTS articles_ai AFTER INSERT ON articles BEGIN