nandi/glean · faac4b3

nandi/gleanpublic⑂ Fork 0

⑂ faac4b3

Handle vector table schema mismatches (happens when changing embedding model)Unverified

Julien Robert committed 2026-04-28T10:07:30+02:00 Browse files

faac4b3 parent: 109590a

modified .env.example +5 -5 Viewed

@@ -11,10 +11,10 @@ GLEAN_BACKFILL_CONCURRENCY=5
11	11	# Leave empty for localhost OAuth (development)
12	12	# GLEAN_OAUTH_CLIENT_ID=https://glean.at/oauth/client-metadata
13	13	# GLEAN_OAUTH_REDIRECT_URL=https://glean.at/auth/callback
14		-# Embeddings (recommended — powers content-based feed/article recommendations)
	14	+# Embeddings (recommended as it powers content-based feed/article recommendations)
15	15	# Point to any OpenAI-compatible /v1/embeddings endpoint (OpenAI, Ollama, etc.)
16	16	# Without embeddings, recommendations rely only on subscription overlap and social graph.
17		-GLEAN_EMBED_BASE_URL=https://api.openai.com/v1
18		-GLEAN_EMBED_API_KEY=sk-...
19		-GLEAN_EMBED_MODEL=text-embedding-3-small
20		-GLEAN_EMBED_DIMENSION=1536
	17	+GLEAN_EMBED_BASE_URL=https://llms.example.com/v1
	18	+GLEAN_EMBED_API_KEY=API-KEY-001
	19	+GLEAN_EMBED_MODEL=qwen-embedding-4b
	20	+GLEAN_EMBED_DIMENSION=2560

  # Leave empty for localhost OAuth (development)
  # GLEAN_OAUTH_CLIENT_ID=https://glean.at/oauth/client-metadata
  # GLEAN_OAUTH_REDIRECT_URL=https://glean.at/auth/callback
--# Embeddings (recommended — powers content-based feed/article recommendations)
++# Embeddings (recommended as it powers content-based feed/article recommendations)
  # Point to any OpenAI-compatible /v1/embeddings endpoint (OpenAI, Ollama, etc.)
  # Without embeddings, recommendations rely only on subscription overlap and social graph.
--GLEAN_EMBED_BASE_URL=https://api.openai.com/v1
++GLEAN_EMBED_BASE_URL=https://llms.example.com/v1
--GLEAN_EMBED_API_KEY=sk-...
++GLEAN_EMBED_API_KEY=API-KEY-001
--GLEAN_EMBED_MODEL=text-embedding-3-small
++GLEAN_EMBED_MODEL=qwen-embedding-4b
--GLEAN_EMBED_DIMENSION=1536
++GLEAN_EMBED_DIMENSION=2560

modified internal/db/db.go +29 -5 Viewed

@@ -131,12 +131,36 @@ func (s *Store) InitVecTables(dimension int) error {
131	131	if dimension <= 0 {
132	132	return nil
133	133	}
134		- for _, stmt := range []string{
135		- fmt.Sprintf(`CREATE VIRTUAL TABLE IF NOT EXISTS recs.feed_embeddings USING vec0(feed_url TEXT PRIMARY KEY, embedding float[%d])`, dimension),
136		- fmt.Sprintf(`CREATE VIRTUAL TABLE IF NOT EXISTS recs.article_embeddings USING vec0(article_id INTEGER PRIMARY KEY, embedding float[%d])`, dimension),
	134	+
	135	+ for _, tbl := range []struct {
	136	+ name string
	137	+ col string
	138	+ create string
	139	+ }{
	140	+ {
	141	+ name: "recs.feed_embeddings",
	142	+ col: "feed_url",
	143	+ create: fmt.Sprintf(`CREATE VIRTUAL TABLE recs.feed_embeddings USING vec0(feed_url TEXT PRIMARY KEY, embedding float[%d])`, dimension),
	144	+ },
	145	+ {
	146	+ name: "recs.article_embeddings",
	147	+ col: "article_id",
	148	+ create: fmt.Sprintf(`CREATE VIRTUAL TABLE recs.article_embeddings USING vec0(article_id INTEGER PRIMARY KEY, embedding float[%d])`, dimension),
	149	+ },
137	150	} {
138		- if _, err := s.db.ExecContext(context.Background(), stmt); err != nil {
139		- return fmt.Errorf("create vec0 table: %w", err)
	151	+ var schema string
	152	+ _ = s.db.QueryRow("SELECT sql FROM recs.sqlite_master WHERE type='table' AND name=?", strings.TrimPrefix(tbl.name, "recs.")).Scan(&schema)
	153	+ expected := fmt.Sprintf("float[%d]", dimension)
	154	+ if strings.Contains(schema, expected) {
	155	+ continue
	156	+ }
	157	+
	158	+ if schema != "" {
	159	+ s.db.Exec(fmt.Sprintf("DROP TABLE %s", tbl.name))
	160	+ }
	161	+
	162	+ if _, err := s.db.ExecContext(context.Background(), tbl.create); err != nil {
	163	+ return fmt.Errorf("create vec0 table %s: %w", tbl.name, err)
140	164	}
141	165	}
142	166	return nil

  	if dimension <= 0 {
  		return nil
  	}
--	for _, stmt := range []string{
++
--		fmt.Sprintf(`CREATE VIRTUAL TABLE IF NOT EXISTS recs.feed_embeddings USING vec0(feed_url TEXT PRIMARY KEY, embedding float[%d])`, dimension),
++	for _, tbl := range []struct {
--		fmt.Sprintf(`CREATE VIRTUAL TABLE IF NOT EXISTS recs.article_embeddings USING vec0(article_id INTEGER PRIMARY KEY, embedding float[%d])`, dimension),
++		name   string
++		col    string
++		create string
++	}{
++		{
++			name:   "recs.feed_embeddings",
++			col:    "feed_url",
++			create: fmt.Sprintf(`CREATE VIRTUAL TABLE recs.feed_embeddings USING vec0(feed_url TEXT PRIMARY KEY, embedding float[%d])`, dimension),
++		},
++		{
++			name:   "recs.article_embeddings",
++			col:    "article_id",
++			create: fmt.Sprintf(`CREATE VIRTUAL TABLE recs.article_embeddings USING vec0(article_id INTEGER PRIMARY KEY, embedding float[%d])`, dimension),
++		},
  	} {
--		if _, err := s.db.ExecContext(context.Background(), stmt); err != nil {
++		var schema string
--			return fmt.Errorf("create vec0 table: %w", err)
++		_ = s.db.QueryRow("SELECT sql FROM recs.sqlite_master WHERE type='table' AND name=?", strings.TrimPrefix(tbl.name, "recs.")).Scan(&schema)
++		expected := fmt.Sprintf("float[%d]", dimension)
++		if strings.Contains(schema, expected) {
++			continue
++		}
++
++		if schema != "" {
++			s.db.Exec(fmt.Sprintf("DROP TABLE %s", tbl.name))
++		}
++
++		if _, err := s.db.ExecContext(context.Background(), tbl.create); err != nil {
++			return fmt.Errorf("create vec0 table %s: %w", tbl.name, err)
  		}
  	}
  	return nil