nandi/glean · 75afa72

modified docs/specs.md +1 -12 Viewed

@@ -377,14 +377,9 @@ CREATE TABLE articles (
377	377	UNIQUE(feed_url, guid)
378	378	);
379	379
380		-CREATE INDEX idx_articles_feed ON articles(feed_url);
381		-CREATE INDEX idx_articles_published ON articles(published DESC);
	380	+CREATE INDEX idx_articles_language ON articles(language);
382	381	```
383	382
384		-Content is stored as raw HTML from the feed's `<content:encoded>`, `<summary>`, or JSON Feed `content_html`. `full_content` stores scraped article content fetched from the original URL. The server renders it in a sanitized view (strip `<script>`, `<iframe>`, etc.).
385		-
386		-The `language` column stores the ISO 639-1 code detected by the LLM (e.g. `en`, `fr`, `ja`). It defaults to empty (`''`) and is populated by the cron job when `GLEAN_LLM_BASE_URL` is configured.
387		-
388	383	### 4.5 Read State
389	384
390	385	Read/unread state is tracked per user per article:
@@ -516,9 +511,6 @@ CREATE TABLE subscriptions (
516	511	cid TEXT,
517	512	UNIQUE(user_did, feed_url)
518	513	);
519		-
520		-CREATE INDEX idx_subscriptions_feed ON subscriptions(feed_url);
521		-CREATE INDEX idx_subscriptions_user ON subscriptions(user_did);
522	514	```
523	515
524	516	### 6.3 Feeds (`<base>_articles`)
@@ -563,8 +555,6 @@ CREATE TABLE articles (
563	555	UNIQUE(feed_url, guid)
564	556	);
565	557
566		-CREATE INDEX idx_articles_feed ON articles(feed_url);
567		-CREATE INDEX idx_articles_published ON articles(published DESC);
568	558	CREATE INDEX idx_articles_language ON articles(language);
569	559	```
570	560
@@ -656,7 +646,6 @@ CREATE TABLE follows (
656	646
657	647	CREATE INDEX idx_follows_target ON follows(target_did);
658	648	CREATE INDEX idx_follows_uri ON follows(uri);
659		-CREATE INDEX idx_follows_followed_at ON follows(followed_at);
660	649	```
661	650
662	651	### 6.9 OAuth Storage (`<base>_users`)

      UNIQUE(feed_url, guid)
  );
--CREATE INDEX idx_articles_feed ON articles(feed_url);
++CREATE INDEX idx_articles_language ON articles(language);
--CREATE INDEX idx_articles_published ON articles(published DESC);
  ```
--Content is stored as raw HTML from the feed's `<content:encoded>`, `<summary>`, or JSON Feed `content_html`. `full_content` stores scraped article content fetched from the original URL. The server renders it in a sanitized view (strip `<script>`, `<iframe>`, etc.).
--
--The `language` column stores the ISO 639-1 code detected by the LLM (e.g. `en`, `fr`, `ja`). It defaults to empty (`''`) and is populated by the cron job when `GLEAN_LLM_BASE_URL` is configured.
--
  ### 4.5 Read State
  Read/unread state is tracked per user per article:
      cid         TEXT,
      UNIQUE(user_did, feed_url)
  );
--
--CREATE INDEX idx_subscriptions_feed ON subscriptions(feed_url);
--CREATE INDEX idx_subscriptions_user ON subscriptions(user_did);
  ```
  ### 6.3 Feeds (`<base>_articles`)
      UNIQUE(feed_url, guid)
  );
--CREATE INDEX idx_articles_feed ON articles(feed_url);
--CREATE INDEX idx_articles_published ON articles(published DESC);
  CREATE INDEX idx_articles_language ON articles(language);
  ```
  CREATE INDEX idx_follows_target ON follows(target_did);
  CREATE INDEX idx_follows_uri ON follows(uri);
--CREATE INDEX idx_follows_followed_at ON follows(followed_at);
  ```
  ### 6.9 OAuth Storage (`<base>_users`)

modified internal/db/db.go +0 -12 Viewed

@@ -252,7 +252,6 @@ var usersSchema = []string{
252	252
253	253	`CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
254	254	`CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
255		- `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
256	255
257	256	`CREATE TABLE IF NOT EXISTS user_settings (
258	257	did TEXT PRIMARY KEY,
@@ -281,9 +280,7 @@ var usersSchema = []string{
281	280	PRIMARY KEY (user_did, target_type, target_id)
282	281	)`,
283	282
284		- `CREATE INDEX IF NOT EXISTS idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`,
285	283	`CREATE INDEX IF NOT EXISTS idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`,
286		- `CREATE INDEX IF NOT EXISTS idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
287	284
288	285	`CREATE TABLE IF NOT EXISTS jetstream_cursor (
289	286	id INTEGER PRIMARY KEY CHECK(id = 1),
@@ -368,21 +365,14 @@ var articlesSchema = []string{
368	365	UNIQUE(author_did, feed_url, article_url)
369	366	)`,
370	367
371		- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`,
372	368	`CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
373		- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`,
374		- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user_feed ON subscriptions(user_did, feed_url)`,
375	369	`CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`,
376	370	`CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
377		- `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`,
378		- `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`,
379	371	`CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`,
380	372	`CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`,
381	373	`CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`,
382	374	`CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`,
383	375	`CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`,
384		- `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`,
385		- `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`,
386	376	`CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`,
387	377	`CREATE INDEX IF NOT EXISTS articles.idx_articles_language ON articles(language)`,
388	378	`CREATE INDEX IF NOT EXISTS articles.idx_likes_article_created ON likes(feed_url, article_url, created_at)`,
@@ -456,10 +446,8 @@ var recsSchema = []string{
456	446	source_text TEXT NOT NULL DEFAULT ''
457	447	)`,
458	448
459		- `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`,
460	449	`CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
461	450	`CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`,
462		- `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`,
463	451	}
464	452
465	453	func NullStr(s string) sql.NullString {

  	`CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
  	`CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
--	`CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
  	`CREATE TABLE IF NOT EXISTS user_settings (
  		did TEXT PRIMARY KEY,
  		PRIMARY KEY (user_did, target_type, target_id)
  	)`,
--	`CREATE INDEX IF NOT EXISTS idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`,
  	`CREATE INDEX IF NOT EXISTS idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`,
--	`CREATE INDEX IF NOT EXISTS idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
  	`CREATE TABLE IF NOT EXISTS jetstream_cursor (
  		id INTEGER PRIMARY KEY CHECK(id = 1),
  		UNIQUE(author_did, feed_url, article_url)
  	)`,
--	`CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
--	`CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`,
--	`CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user_feed ON subscriptions(user_did, feed_url)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
--	`CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`,
--	`CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`,
--	`CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`,
--	`CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_articles_language ON articles(language)`,
  	`CREATE INDEX IF NOT EXISTS articles.idx_likes_article_created ON likes(feed_url, article_url, created_at)`,
  		source_text TEXT NOT NULL DEFAULT ''
  	)`,
--	`CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`,
  	`CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
  	`CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`,
--	`CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`,
  }
  func NullStr(s string) sql.NullString {

modified internal/db/migrations.go +40 -9 Viewed

@@ -255,14 +255,45 @@ func migrateUserSettingsExpandedView(db *DB) error {
255	255	}
256	256
257	257	func migrateDropFollowsUserIndex(db *DB) error {
258		- var name string
259		- err := db.QueryRow("SELECT name FROM sqlite_master WHERE type='index' AND name='idx_follows_user'").Scan(&name)
260		- if err == sql.ErrNoRows {
261		- return nil
262		- }
263		- if err != nil {
264		- return fmt.Errorf("check idx_follows_user: %w", err)
	258	+ for _, idx := range []string{
	259	+ "idx_follows_user",
	260	+ "idx_follows_followed_at",
	261	+ "idx_dismissed_user_type",
	262	+ "idx_impressions_last_shown",
	263	+ "idx_subscriptions_feed",
	264	+ "idx_subscriptions_user",
	265	+ "idx_subscriptions_user_feed",
	266	+ "idx_articles_feed",
	267	+ "idx_articles_published",
	268	+ "idx_likes_article",
	269	+ "idx_likes_author",
	270	+ "idx_follow_distances_b",
	271	+ "idx_user_similarity_a",
	272	+ } {
	273	+ var schema string
	274	+ if strings.HasPrefix(idx, "idx_subscriptions_") \|\| strings.HasPrefix(idx, "idx_articles_") \|\| strings.HasPrefix(idx, "idx_likes_") {
	275	+ _ = db.QueryRow("SELECT name FROM articles.sqlite_master WHERE type='index' AND name=?", idx).Scan(&schema)
	276	+ } else if strings.HasPrefix(idx, "idx_follow_distances_") \|\| strings.HasPrefix(idx, "idx_user_similarity_") {
	277	+ _ = db.QueryRow("SELECT name FROM recs.sqlite_master WHERE type='index' AND name=?", idx).Scan(&schema)
	278	+ } else {
	279	+ _ = db.QueryRow("SELECT name FROM sqlite_master WHERE type='index' AND name=?", idx).Scan(&schema)
	280	+ }
	281	+ if schema == "" {
	282	+ continue
	283	+ }
	284	+ if strings.HasPrefix(idx, "idx_subscriptions_") \|\| strings.HasPrefix(idx, "idx_articles_") \|\| strings.HasPrefix(idx, "idx_likes_") {
	285	+ if _, err := db.Exec(fmt.Sprintf("DROP INDEX IF EXISTS articles.%s", idx)); err != nil {
	286	+ return fmt.Errorf("drop %s: %w", idx, err)
	287	+ }
	288	+ } else if strings.HasPrefix(idx, "idx_follow_distances_") \|\| strings.HasPrefix(idx, "idx_user_similarity_") {
	289	+ if _, err := db.Exec(fmt.Sprintf("DROP INDEX IF EXISTS recs.%s", idx)); err != nil {
	290	+ return fmt.Errorf("drop %s: %w", idx, err)
	291	+ }
	292	+ } else {
	293	+ if _, err := db.Exec(fmt.Sprintf("DROP INDEX IF EXISTS %s", idx)); err != nil {
	294	+ return fmt.Errorf("drop %s: %w", idx, err)
	295	+ }
	296	+ }
265	297	}
266		- _, err = db.Exec("DROP INDEX IF EXISTS idx_follows_user")
267		- return err
	298	+ return nil
268	299	}

  }
  func migrateDropFollowsUserIndex(db *DB) error {
--	var name string
++	for _, idx := range []string{
--	err := db.QueryRow("SELECT name FROM sqlite_master WHERE type='index' AND name='idx_follows_user'").Scan(&name)
++		"idx_follows_user",
--	if err == sql.ErrNoRows {
++		"idx_follows_followed_at",
--		return nil
++		"idx_dismissed_user_type",
--	}
++		"idx_impressions_last_shown",
--	if err != nil {
++		"idx_subscriptions_feed",
--		return fmt.Errorf("check idx_follows_user: %w", err)
++		"idx_subscriptions_user",
++		"idx_subscriptions_user_feed",
++		"idx_articles_feed",
++		"idx_articles_published",
++		"idx_likes_article",
++		"idx_likes_author",
++		"idx_follow_distances_b",
++		"idx_user_similarity_a",
++	} {
++		var schema string
++		if strings.HasPrefix(idx, "idx_subscriptions_") || strings.HasPrefix(idx, "idx_articles_") || strings.HasPrefix(idx, "idx_likes_") {
++			_ = db.QueryRow("SELECT name FROM articles.sqlite_master WHERE type='index' AND name=?", idx).Scan(&schema)
++		} else if strings.HasPrefix(idx, "idx_follow_distances_") || strings.HasPrefix(idx, "idx_user_similarity_") {
++			_ = db.QueryRow("SELECT name FROM recs.sqlite_master WHERE type='index' AND name=?", idx).Scan(&schema)
++		} else {
++			_ = db.QueryRow("SELECT name FROM sqlite_master WHERE type='index' AND name=?", idx).Scan(&schema)
++		}
++		if schema == "" {
++			continue
++		}
++		if strings.HasPrefix(idx, "idx_subscriptions_") || strings.HasPrefix(idx, "idx_articles_") || strings.HasPrefix(idx, "idx_likes_") {
++			if _, err := db.Exec(fmt.Sprintf("DROP INDEX IF EXISTS articles.%s", idx)); err != nil {
++				return fmt.Errorf("drop %s: %w", idx, err)
++			}
++		} else if strings.HasPrefix(idx, "idx_follow_distances_") || strings.HasPrefix(idx, "idx_user_similarity_") {
++			if _, err := db.Exec(fmt.Sprintf("DROP INDEX IF EXISTS recs.%s", idx)); err != nil {
++				return fmt.Errorf("drop %s: %w", idx, err)
++			}
++		} else {
++			if _, err := db.Exec(fmt.Sprintf("DROP INDEX IF EXISTS %s", idx)); err != nil {
++				return fmt.Errorf("drop %s: %w", idx, err)
++			}
++		}
  	}
--	_, err = db.Exec("DROP INDEX IF EXISTS idx_follows_user")
++	return nil
--	return err
  }

Remove unused database indexes and add migration to drop themUnverified