rickub
nandi/gleanpublic⑂ Fork 0
⑂ 39d8617
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Optimize database queries with indexes and temp tablesUnverified

Julien Robert committed 2026-04-22T13:51:11+02:00 Browse files
39d8617 parent: 1358397
modified internal/cluster/jaccard.go +6 -0
@@ -133,6 +133,12 @@ func (e *Engine) computeDescriptionSimilarity(ctx context.Context, tx *sql.Tx) e
133133 return err
134134 }
135135
136+ if _, err := tx.ExecContext(ctx, `
137+ CREATE INDEX IF NOT EXISTS _idx_feed_words_word ON _feed_words(word, feed_url)
138+ `); err != nil {
139+ return err
140+ }
141+
136142 descInsert := `
137143 INSERT OR IGNORE INTO feed_similarity (feed_a, feed_b, jaccard)
138144 SELECT feed_a, feed_b, 0 FROM _word_overlap
@@ -133,6 +133,12 @@ func (e *Engine) computeDescriptionSimilarity(ctx context.Context, tx *sql.Tx) e
133 return err133 return err
134 }134 }
135 135
136+ if _, err := tx.ExecContext(ctx, `
137+ CREATE INDEX IF NOT EXISTS _idx_feed_words_word ON _feed_words(word, feed_url)
138+ `); err != nil {
139+ return err
140+ }
141+
136 descInsert := `142 descInsert := `
137 INSERT OR IGNORE INTO feed_similarity (feed_a, feed_b, jaccard)143 INSERT OR IGNORE INTO feed_similarity (feed_a, feed_b, jaccard)
138 SELECT feed_a, feed_b, 0 FROM _word_overlap144 SELECT feed_a, feed_b, 0 FROM _word_overlap
modified internal/cluster/scoring.go +64 -12
@@ -286,23 +286,75 @@ func (e *Engine) ComputeSignalProfiles(ctx context.Context) error {
286286 return err
287287 }
288288
289+ if _, err := tx.ExecContext(ctx, `
290+ CREATE TEMP TABLE IF NOT EXISTS _user_like_counts (user_did TEXT PRIMARY KEY, cnt INT)
291+ `); err != nil {
292+ return err
293+ }
294+ if _, err := tx.ExecContext(ctx, `DELETE FROM _user_like_counts`); err != nil {
295+ return err
296+ }
297+ if _, err := tx.ExecContext(ctx, `
298+ INSERT INTO _user_like_counts SELECT author_did, COUNT(*) FROM likes GROUP BY author_did
299+ `); err != nil {
300+ return err
301+ }
302+
303+ if _, err := tx.ExecContext(ctx, `
304+ CREATE TEMP TABLE IF NOT EXISTS _user_tag_counts (user_did TEXT PRIMARY KEY, cnt INT)
305+ `); err != nil {
306+ return err
307+ }
308+ if _, err := tx.ExecContext(ctx, `DELETE FROM _user_tag_counts`); err != nil {
309+ return err
310+ }
311+ if _, err := tx.ExecContext(ctx, `
312+ INSERT INTO _user_tag_counts
313+ WITH user_tags AS (
314+ SELECT author_did, TRIM(value) AS tag
315+ FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')
316+ WHERE tags IS NOT NULL AND tags != ''
317+ )
318+ SELECT author_did, COUNT(DISTINCT tag) FROM user_tags GROUP BY author_did
319+ `); err != nil {
320+ return err
321+ }
322+
323+ if _, err := tx.ExecContext(ctx, `
324+ CREATE TEMP TABLE IF NOT EXISTS _user_top_categories (user_did TEXT PRIMARY KEY, categories TEXT)
325+ `); err != nil {
326+ return err
327+ }
328+ if _, err := tx.ExecContext(ctx, `DELETE FROM _user_top_categories`); err != nil {
329+ return err
330+ }
331+ if _, err := tx.ExecContext(ctx, `
332+ INSERT INTO _user_top_categories
333+ SELECT user_did, '[' || GROUP_CONCAT('{"c":"' || category || '","n":"' || CAST(cnt AS TEXT) || '}') || ']'
334+ FROM (
335+ SELECT user_did, category, COUNT(*) AS cnt
336+ FROM subscriptions
337+ WHERE category IS NOT NULL AND category != ''
338+ GROUP BY user_did, category
339+ ORDER BY COUNT(*) DESC
340+ LIMIT 5
341+ )
342+ GROUP BY user_did
343+ `); err != nil {
344+ return err
345+ }
346+
289347 _, err = tx.ExecContext(ctx, `
290348 INSERT INTO user_signal_profiles (user_did, total_likes, total_tags, top_categories)
291349 SELECT
292350 u.did,
293- (SELECT COUNT(*) FROM likes WHERE author_did = u.did),
294- COALESCE((SELECT COUNT(DISTINCT TRIM(value))
295- FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')
296- WHERE author_did = u.did AND tags IS NOT NULL AND tags != ''
297- ), 0),
298- (SELECT '[' || GROUP_CONCAT('{"c":"' || category || '","n":"' || CAST(cnt AS TEXT) || '}') || ']'
299- FROM (
300- SELECT category, COUNT(*) AS cnt
301- FROM subscriptions WHERE user_did = u.did AND category IS NOT NULL AND category != ''
302- GROUP BY category ORDER BY cnt DESC LIMIT 5
303- )
304- )
351+ COALESCE(lc.cnt, 0),
352+ COALESCE(tc.cnt, 0),
353+ COALESCE(cc.categories, '[]')
305354 FROM users u
355+ LEFT JOIN _user_like_counts lc ON lc.user_did = u.did
356+ LEFT JOIN _user_tag_counts tc ON tc.user_did = u.did
357+ LEFT JOIN _user_top_categories cc ON cc.user_did = u.did
306358 `)
307359 if err != nil {
308360 return err
@@ -286,23 +286,75 @@ func (e *Engine) ComputeSignalProfiles(ctx context.Context) error {
286 return err286 return err
287 }287 }
288 288
289+ if _, err := tx.ExecContext(ctx, `
290+ CREATE TEMP TABLE IF NOT EXISTS _user_like_counts (user_did TEXT PRIMARY KEY, cnt INT)
291+ `); err != nil {
292+ return err
293+ }
294+ if _, err := tx.ExecContext(ctx, `DELETE FROM _user_like_counts`); err != nil {
295+ return err
296+ }
297+ if _, err := tx.ExecContext(ctx, `
298+ INSERT INTO _user_like_counts SELECT author_did, COUNT(*) FROM likes GROUP BY author_did
299+ `); err != nil {
300+ return err
301+ }
302+
303+ if _, err := tx.ExecContext(ctx, `
304+ CREATE TEMP TABLE IF NOT EXISTS _user_tag_counts (user_did TEXT PRIMARY KEY, cnt INT)
305+ `); err != nil {
306+ return err
307+ }
308+ if _, err := tx.ExecContext(ctx, `DELETE FROM _user_tag_counts`); err != nil {
309+ return err
310+ }
311+ if _, err := tx.ExecContext(ctx, `
312+ INSERT INTO _user_tag_counts
313+ WITH user_tags AS (
314+ SELECT author_did, TRIM(value) AS tag
315+ FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')
316+ WHERE tags IS NOT NULL AND tags != ''
317+ )
318+ SELECT author_did, COUNT(DISTINCT tag) FROM user_tags GROUP BY author_did
319+ `); err != nil {
320+ return err
321+ }
322+
323+ if _, err := tx.ExecContext(ctx, `
324+ CREATE TEMP TABLE IF NOT EXISTS _user_top_categories (user_did TEXT PRIMARY KEY, categories TEXT)
325+ `); err != nil {
326+ return err
327+ }
328+ if _, err := tx.ExecContext(ctx, `DELETE FROM _user_top_categories`); err != nil {
329+ return err
330+ }
331+ if _, err := tx.ExecContext(ctx, `
332+ INSERT INTO _user_top_categories
333+ SELECT user_did, '[' || GROUP_CONCAT('{"c":"' || category || '","n":"' || CAST(cnt AS TEXT) || '}') || ']'
334+ FROM (
335+ SELECT user_did, category, COUNT(*) AS cnt
336+ FROM subscriptions
337+ WHERE category IS NOT NULL AND category != ''
338+ GROUP BY user_did, category
339+ ORDER BY COUNT(*) DESC
340+ LIMIT 5
341+ )
342+ GROUP BY user_did
343+ `); err != nil {
344+ return err
345+ }
346+
289 _, err = tx.ExecContext(ctx, `347 _, err = tx.ExecContext(ctx, `
290 INSERT INTO user_signal_profiles (user_did, total_likes, total_tags, top_categories)348 INSERT INTO user_signal_profiles (user_did, total_likes, total_tags, top_categories)
291 SELECT349 SELECT
292 u.did,350 u.did,
293- (SELECT COUNT(*) FROM likes WHERE author_did = u.did),351+ COALESCE(lc.cnt, 0),
294- COALESCE((SELECT COUNT(DISTINCT TRIM(value))352+ COALESCE(tc.cnt, 0),
295- FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')353+ COALESCE(cc.categories, '[]')
296- WHERE author_did = u.did AND tags IS NOT NULL AND tags != ''
297- ), 0),
298- (SELECT '[' || GROUP_CONCAT('{"c":"' || category || '","n":"' || CAST(cnt AS TEXT) || '}') || ']'
299- FROM (
300- SELECT category, COUNT(*) AS cnt
301- FROM subscriptions WHERE user_did = u.did AND category IS NOT NULL AND category != ''
302- GROUP BY category ORDER BY cnt DESC LIMIT 5
303- )
304- )
305 FROM users u354 FROM users u
355+ LEFT JOIN _user_like_counts lc ON lc.user_did = u.did
356+ LEFT JOIN _user_tag_counts tc ON tc.user_did = u.did
357+ LEFT JOIN _user_top_categories cc ON cc.user_did = u.did
306 `)358 `)
307 if err != nil {359 if err != nil {
308 return err360 return err
modified internal/cluster/social.go +5 -11
@@ -15,22 +15,16 @@ func (e *Engine) ComputeFollowDistances(ctx context.Context) error {
1515 return err
1616 }
1717
18- if _, err := tx.ExecContext(ctx, `
18+ _, err = tx.ExecContext(ctx, `
1919 INSERT INTO follow_distances (user_a, user_b, distance)
20- SELECT user_did, target_did, 1
21- FROM follows
22- WHERE user_did != target_did
23- `); err != nil {
24- return err
25- }
26-
27- if _, err := tx.ExecContext(ctx, `
28- INSERT OR IGNORE INTO follow_distances (user_a, user_b, distance)
20+ SELECT user_did, target_did, 1 FROM follows WHERE user_did != target_did
21+ UNION ALL
2922 SELECT f1.user_did, f2.target_did, 2
3023 FROM follows f1
3124 JOIN follows f2 ON f1.target_did = f2.user_did
3225 WHERE f1.user_did != f2.target_did
33- `); err != nil {
26+ `)
27+ if err != nil {
3428 return err
3529 }
3630
@@ -15,22 +15,16 @@ func (e *Engine) ComputeFollowDistances(ctx context.Context) error {
15 return err15 return err
16 }16 }
17 17
18- if _, err := tx.ExecContext(ctx, `18+ _, err = tx.ExecContext(ctx, `
19 INSERT INTO follow_distances (user_a, user_b, distance)19 INSERT INTO follow_distances (user_a, user_b, distance)
20- SELECT user_did, target_did, 120+ SELECT user_did, target_did, 1 FROM follows WHERE user_did != target_did
21- FROM follows21+ UNION ALL
22- WHERE user_did != target_did
23- `); err != nil {
24- return err
25- }
26-
27- if _, err := tx.ExecContext(ctx, `
28- INSERT OR IGNORE INTO follow_distances (user_a, user_b, distance)
29 SELECT f1.user_did, f2.target_did, 222 SELECT f1.user_did, f2.target_did, 2
30 FROM follows f123 FROM follows f1
31 JOIN follows f2 ON f1.target_did = f2.user_did24 JOIN follows f2 ON f1.target_did = f2.user_did
32 WHERE f1.user_did != f2.target_did25 WHERE f1.user_did != f2.target_did
33- `); err != nil {26+ `)
27+ if err != nil {
34 return err28 return err
35 }29 }
36 30
modified internal/db/db.go +3 -1
@@ -207,8 +207,10 @@ var schema = []string{
207207 PRIMARY KEY (account_did, session_id)
208208 )`,
209209 `CREATE INDEX IF NOT EXISTS idx_subscriptions_feed ON subscriptions(feed_url)`,
210+ `CREATE INDEX IF NOT EXISTS idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
210211 `CREATE INDEX IF NOT EXISTS idx_subscriptions_user ON subscriptions(user_did)`,
211212 `CREATE INDEX IF NOT EXISTS idx_subscriptions_uri ON subscriptions(uri)`,
213+ `CREATE INDEX IF NOT EXISTS idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
212214 `CREATE INDEX IF NOT EXISTS idx_articles_feed ON articles(feed_url)`,
213215 `CREATE INDEX IF NOT EXISTS idx_articles_published ON articles(published DESC)`,
214216 `CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(url)`,
@@ -223,6 +225,7 @@ var schema = []string{
223225 `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
224226 `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
225227 `CREATE INDEX IF NOT EXISTS idx_user_similarity_b ON user_similarity(user_b)`,
228+ `CREATE INDEX IF NOT EXISTS idx_user_similarity_a ON user_similarity(user_a)`,
226229
227230 `CREATE TABLE IF NOT EXISTS dismissed_recommendations (
228231 user_did TEXT NOT NULL REFERENCES users(did),
@@ -275,7 +278,6 @@ var schema = []string{
275278 `CREATE INDEX IF NOT EXISTS idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
276279 `CREATE INDEX IF NOT EXISTS idx_follow_distances_b ON follow_distances(user_b)`,
277280 `CREATE INDEX IF NOT EXISTS idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
278- `CREATE INDEX IF NOT EXISTS idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
279281 `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
280282 `CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle)`,
281283 `CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,
@@ -207,8 +207,10 @@ var schema = []string{
207 PRIMARY KEY (account_did, session_id)207 PRIMARY KEY (account_did, session_id)
208 )`,208 )`,
209 `CREATE INDEX IF NOT EXISTS idx_subscriptions_feed ON subscriptions(feed_url)`,209 `CREATE INDEX IF NOT EXISTS idx_subscriptions_feed ON subscriptions(feed_url)`,
210+ `CREATE INDEX IF NOT EXISTS idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
210 `CREATE INDEX IF NOT EXISTS idx_subscriptions_user ON subscriptions(user_did)`,211 `CREATE INDEX IF NOT EXISTS idx_subscriptions_user ON subscriptions(user_did)`,
211 `CREATE INDEX IF NOT EXISTS idx_subscriptions_uri ON subscriptions(uri)`,212 `CREATE INDEX IF NOT EXISTS idx_subscriptions_uri ON subscriptions(uri)`,
213+ `CREATE INDEX IF NOT EXISTS idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
212 `CREATE INDEX IF NOT EXISTS idx_articles_feed ON articles(feed_url)`,214 `CREATE INDEX IF NOT EXISTS idx_articles_feed ON articles(feed_url)`,
213 `CREATE INDEX IF NOT EXISTS idx_articles_published ON articles(published DESC)`,215 `CREATE INDEX IF NOT EXISTS idx_articles_published ON articles(published DESC)`,
214 `CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(url)`,216 `CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(url)`,
@@ -223,6 +225,7 @@ var schema = []string{
223 `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,225 `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
224 `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,226 `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
225 `CREATE INDEX IF NOT EXISTS idx_user_similarity_b ON user_similarity(user_b)`,227 `CREATE INDEX IF NOT EXISTS idx_user_similarity_b ON user_similarity(user_b)`,
228+ `CREATE INDEX IF NOT EXISTS idx_user_similarity_a ON user_similarity(user_a)`,
226 229
227 `CREATE TABLE IF NOT EXISTS dismissed_recommendations (230 `CREATE TABLE IF NOT EXISTS dismissed_recommendations (
228 user_did TEXT NOT NULL REFERENCES users(did),231 user_did TEXT NOT NULL REFERENCES users(did),
@@ -275,7 +278,6 @@ var schema = []string{
275 `CREATE INDEX IF NOT EXISTS idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,278 `CREATE INDEX IF NOT EXISTS idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
276 `CREATE INDEX IF NOT EXISTS idx_follow_distances_b ON follow_distances(user_b)`,279 `CREATE INDEX IF NOT EXISTS idx_follow_distances_b ON follow_distances(user_b)`,
277 `CREATE INDEX IF NOT EXISTS idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,280 `CREATE INDEX IF NOT EXISTS idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
278- `CREATE INDEX IF NOT EXISTS idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
279 `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,281 `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
280 `CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle)`,282 `CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle)`,
281 `CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,283 `CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,