rickub
nandi/gleanpublic Fork
⑂ 51371bc
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Download ZIP

Port the database layer's foundation

Opens storage the way the Go backend does: <base>_users as main, with
_articles and _recs attached, so queries join across them while each file
checkpoints and vacuums alone -- and so the derived recommendation tables
can be rebuilt without touching anything a user wrote. The suffixes match,
so a database written by either side is readable by the other.

Pragmas are repeated per schema rather than set once. PRAGMA applies to one
database at a time, so setting WAL on main leaves articles journalling the
slow way; the probe asserts WAL on all three rather than trusting the call.

exp() and log() are registered as SQL functions as the Go connection does.
Nothing calls them today -- they exist so a scoring query can decay a weight
without pulling rows into the application. log() returns NULL for zero and
negatives, since -inf and NaN both poison an ORDER BY.

The schema is extracted rather than retyped: tools/extract_schema.py pulls
all 44 DDL statements out of internal/db/db.go. Transcribing them by hand
would drift, and a column differing by a default or a CHECK surfaces months
later as a violation nobody can place.

The probe targets what fails quietly -- an ATTACH that did not happen, a
pragma that did not take, FTS5 triggers that never fire. The last is the
sharpest: the index is trigger-maintained, so a missed trigger means search
returns nothing while every other query looks fine.

This is the foundation, not the layer. The remaining ~2,800 lines of query
methods are mechanical against this base.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
nandithebull committed 2026-09-21T00:24:15-07:00 Browse files
51371bc parent: 1971b41
modified spike/nim/.gitignore +1 -0
@@ -10,3 +10,4 @@ store_probe
1010 *.db
1111 *.db-wal
1212 *.db-shm
13+gleandb_probe
@@ -10,3 +10,4 @@ store_probe
10 *.db10 *.db
11 *.db-wal11 *.db-wal
12 *.db-shm12 *.db-shm
13+gleandb_probe
added spike/nim/atproto/gleandb.nim +147 -0
new file mode 100644
@@ -0,0 +1,147 @@
1+## Glean's database: three SQLite files behind one connection.
2+##
3+## `main` holds users and auth, `articles` the feeds and their contents, and
4+## `recs` the derived recommendation tables. They are separate files attached
5+## together, so a query can join across them while each still checkpoints and
6+## vacuums on its own -- and, more practically, so the derived tables can be
7+## deleted and rebuilt without touching anything a user wrote.
8+##
9+## The pragmas are copied from internal/db/db.go. They are not decoration:
10+## WAL is what lets readers run during a write, and the per-database repeats
11+## matter because PRAGMA applies to one schema at a time -- setting WAL on
12+## `main` leaves `articles` journalling the slow way.
13+
14+import std/[math, options, strformat, strutils]
15+import ./schema
16+import ./sqlite
17+
18+type
19+ GleanDb* = object
20+ db*: Db
21+ basePath*: string
22+
23+{.push importc, cdecl.}
24+proc sqlite3_create_function_v2(db: pointer, name: cstring, nArg: cint,
25+ eTextRep: cint, pApp: pointer,
26+ xFunc, xStep, xFinal, xDestroy: pointer): cint
27+proc sqlite3_value_double(v: pointer): cdouble
28+proc sqlite3_result_double(ctx: pointer, v: cdouble)
29+proc sqlite3_result_null(ctx: pointer)
30+{.pop.}
31+
32+const
33+ SQLITE_UTF8 = 1.cint
34+ SQLITE_DETERMINISTIC = 0x000000800.cint
35+
36+# Registered because the Go connection registers them. Nothing in the current
37+# SQL calls either -- they exist so a scoring query can decay a weight
38+# exponentially without pulling the rows into the application first.
39+proc sqlExp(ctx: pointer, n: cint, args: ptr UncheckedArray[pointer])
40+ {.cdecl.} =
41+ if n != 1:
42+ sqlite3_result_null(ctx)
43+ return
44+ sqlite3_result_double(ctx, exp(sqlite3_value_double(args[0])).cdouble)
45+
46+proc sqlLog(ctx: pointer, n: cint, args: ptr UncheckedArray[pointer])
47+ {.cdecl.} =
48+ if n != 1:
49+ sqlite3_result_null(ctx)
50+ return
51+ let v = sqlite3_value_double(args[0]).float
52+ # log(0) is -inf and log(negative) is NaN; SQL callers want neither, and a
53+ # NULL propagates through an ORDER BY far more gracefully.
54+ if v <= 0:
55+ sqlite3_result_null(ctx)
56+ else:
57+ sqlite3_result_double(ctx, ln(v).cdouble)
58+
59+proc registerFunctions*(db: Db) =
60+ for (name, fn) in [("exp", sqlExp), ("log", sqlLog)]:
61+ let rc = sqlite3_create_function_v2(
62+ db.handle, name.cstring, 1,
63+ SQLITE_UTF8 or SQLITE_DETERMINISTIC,
64+ nil, cast[pointer](fn), nil, nil, nil)
65+ if rc != SQLITE_OK:
66+ raise newException(SqliteError, "registering SQL function " & name)
67+
68+const
69+ MainPragmas = [
70+ "journal_mode=WAL",
71+ "wal_autocheckpoint=1000",
72+ "busy_timeout=30000",
73+ "synchronous=NORMAL",
74+ "temp_store=FILE",
75+ "mmap_size=268435456",
76+ "auto_vacuum=INCREMENTAL",
77+ ]
78+ AttachedPragmas = [
79+ "journal_mode=WAL",
80+ "wal_autocheckpoint=1000",
81+ "busy_timeout=30000",
82+ "synchronous=NORMAL",
83+ "temp_store=FILE",
84+ "mmap_size=268435456",
85+ ]
86+
87+proc applyPragmas(db: Db, schemaName: string, pragmas: openArray[string]) =
88+ for pragma in pragmas:
89+ # A pragma that will not take is not worth failing a boot over -- some are
90+ # advisory, and mmap_size in particular is refused on filesystems that
91+ # cannot map. The ones that matter are asserted after opening instead.
92+ try:
93+ if schemaName.len == 0:
94+ db.exec &"PRAGMA {pragma}"
95+ else:
96+ db.exec &"PRAGMA {schemaName}.{pragma}"
97+ except SqliteError:
98+ discard
99+
100+proc open*(basePath: string): GleanDb =
101+ ## Open `<basePath>_users` and attach `_articles` and `_recs`.
102+ ##
103+ ## The suffixes match the Go layout, so a database written by either side is
104+ ## readable by the other.
105+ let
106+ usersPath = basePath & "_users"
107+ articlesPath = basePath & "_articles"
108+ recsPath = basePath & "_recs"
109+
110+ var db = sqlite.open(usersPath)
111+ registerFunctions(db)
112+ applyPragmas(db, "", MainPragmas)
113+
114+ # Quoted and single-quote-escaped: a path is not necessarily tame, and this
115+ # one can come from configuration.
116+ for (name, path) in [("articles", articlesPath), ("recs", recsPath)]:
117+ db.exec &"ATTACH DATABASE '{path.replace(\"'\", \"''\")}' AS {name}"
118+ applyPragmas(db, name, AttachedPragmas)
119+
120+ GleanDb(db: db, basePath: basePath)
121+
122+proc migrate*(g: GleanDb) =
123+ ## Apply the schema. Every statement is IF NOT EXISTS, so this runs on every
124+ ## boot rather than being gated on a version.
125+ for stmt in usersSchema: g.db.exec stmt
126+ for stmt in articlesSchema: g.db.exec stmt
127+ for stmt in recsSchema: g.db.exec stmt
128+
129+proc createEmbeddingTables*(g: GleanDb, dimension: int) =
130+ ## The vec0 tables carry their dimension in the DDL, so they cannot live in
131+ ## the static schema -- it comes from GLEAN_EMBED_DIMENSION.
132+ g.db.exec &"""CREATE VIRTUAL TABLE IF NOT EXISTS recs.feed_embeddings
133+ USING vec0(feed_url TEXT PRIMARY KEY, embedding float[{dimension}])"""
134+ g.db.exec &"""CREATE VIRTUAL TABLE IF NOT EXISTS recs.article_embeddings
135+ USING vec0(article_id INTEGER PRIMARY KEY, embedding float[{dimension}])"""
136+
137+proc close*(g: var GleanDb) =
138+ g.db.close()
139+
140+proc journalMode*(g: GleanDb, schemaName = ""): string =
141+ let sql = if schemaName.len == 0: "PRAGMA journal_mode"
142+ else: &"PRAGMA {schemaName}.journal_mode"
143+ g.db.queryText(sql).get("")
144+
145+proc attachedSchemas*(g: GleanDb): seq[string] =
146+ for row in g.db.rows("PRAGMA database_list"):
147+ result.add row[1]
new file mode 100644
@@ -0,0 +1,147 @@
1+## Glean's database: three SQLite files behind one connection.
2+##
3+## `main` holds users and auth, `articles` the feeds and their contents, and
4+## `recs` the derived recommendation tables. They are separate files attached
5+## together, so a query can join across them while each still checkpoints and
6+## vacuums on its own -- and, more practically, so the derived tables can be
7+## deleted and rebuilt without touching anything a user wrote.
8+##
9+## The pragmas are copied from internal/db/db.go. They are not decoration:
10+## WAL is what lets readers run during a write, and the per-database repeats
11+## matter because PRAGMA applies to one schema at a time -- setting WAL on
12+## `main` leaves `articles` journalling the slow way.
13+
14+import std/[math, options, strformat, strutils]
15+import ./schema
16+import ./sqlite
17+
18+type
19+ GleanDb* = object
20+ db*: Db
21+ basePath*: string
22+
23+{.push importc, cdecl.}
24+proc sqlite3_create_function_v2(db: pointer, name: cstring, nArg: cint,
25+ eTextRep: cint, pApp: pointer,
26+ xFunc, xStep, xFinal, xDestroy: pointer): cint
27+proc sqlite3_value_double(v: pointer): cdouble
28+proc sqlite3_result_double(ctx: pointer, v: cdouble)
29+proc sqlite3_result_null(ctx: pointer)
30+{.pop.}
31+
32+const
33+ SQLITE_UTF8 = 1.cint
34+ SQLITE_DETERMINISTIC = 0x000000800.cint
35+
36+# Registered because the Go connection registers them. Nothing in the current
37+# SQL calls either -- they exist so a scoring query can decay a weight
38+# exponentially without pulling the rows into the application first.
39+proc sqlExp(ctx: pointer, n: cint, args: ptr UncheckedArray[pointer])
40+ {.cdecl.} =
41+ if n != 1:
42+ sqlite3_result_null(ctx)
43+ return
44+ sqlite3_result_double(ctx, exp(sqlite3_value_double(args[0])).cdouble)
45+
46+proc sqlLog(ctx: pointer, n: cint, args: ptr UncheckedArray[pointer])
47+ {.cdecl.} =
48+ if n != 1:
49+ sqlite3_result_null(ctx)
50+ return
51+ let v = sqlite3_value_double(args[0]).float
52+ # log(0) is -inf and log(negative) is NaN; SQL callers want neither, and a
53+ # NULL propagates through an ORDER BY far more gracefully.
54+ if v <= 0:
55+ sqlite3_result_null(ctx)
56+ else:
57+ sqlite3_result_double(ctx, ln(v).cdouble)
58+
59+proc registerFunctions*(db: Db) =
60+ for (name, fn) in [("exp", sqlExp), ("log", sqlLog)]:
61+ let rc = sqlite3_create_function_v2(
62+ db.handle, name.cstring, 1,
63+ SQLITE_UTF8 or SQLITE_DETERMINISTIC,
64+ nil, cast[pointer](fn), nil, nil, nil)
65+ if rc != SQLITE_OK:
66+ raise newException(SqliteError, "registering SQL function " & name)
67+
68+const
69+ MainPragmas = [
70+ "journal_mode=WAL",
71+ "wal_autocheckpoint=1000",
72+ "busy_timeout=30000",
73+ "synchronous=NORMAL",
74+ "temp_store=FILE",
75+ "mmap_size=268435456",
76+ "auto_vacuum=INCREMENTAL",
77+ ]
78+ AttachedPragmas = [
79+ "journal_mode=WAL",
80+ "wal_autocheckpoint=1000",
81+ "busy_timeout=30000",
82+ "synchronous=NORMAL",
83+ "temp_store=FILE",
84+ "mmap_size=268435456",
85+ ]
86+
87+proc applyPragmas(db: Db, schemaName: string, pragmas: openArray[string]) =
88+ for pragma in pragmas:
89+ # A pragma that will not take is not worth failing a boot over -- some are
90+ # advisory, and mmap_size in particular is refused on filesystems that
91+ # cannot map. The ones that matter are asserted after opening instead.
92+ try:
93+ if schemaName.len == 0:
94+ db.exec &"PRAGMA {pragma}"
95+ else:
96+ db.exec &"PRAGMA {schemaName}.{pragma}"
97+ except SqliteError:
98+ discard
99+
100+proc open*(basePath: string): GleanDb =
101+ ## Open `<basePath>_users` and attach `_articles` and `_recs`.
102+ ##
103+ ## The suffixes match the Go layout, so a database written by either side is
104+ ## readable by the other.
105+ let
106+ usersPath = basePath & "_users"
107+ articlesPath = basePath & "_articles"
108+ recsPath = basePath & "_recs"
109+
110+ var db = sqlite.open(usersPath)
111+ registerFunctions(db)
112+ applyPragmas(db, "", MainPragmas)
113+
114+ # Quoted and single-quote-escaped: a path is not necessarily tame, and this
115+ # one can come from configuration.
116+ for (name, path) in [("articles", articlesPath), ("recs", recsPath)]:
117+ db.exec &"ATTACH DATABASE '{path.replace(\"'\", \"''\")}' AS {name}"
118+ applyPragmas(db, name, AttachedPragmas)
119+
120+ GleanDb(db: db, basePath: basePath)
121+
122+proc migrate*(g: GleanDb) =
123+ ## Apply the schema. Every statement is IF NOT EXISTS, so this runs on every
124+ ## boot rather than being gated on a version.
125+ for stmt in usersSchema: g.db.exec stmt
126+ for stmt in articlesSchema: g.db.exec stmt
127+ for stmt in recsSchema: g.db.exec stmt
128+
129+proc createEmbeddingTables*(g: GleanDb, dimension: int) =
130+ ## The vec0 tables carry their dimension in the DDL, so they cannot live in
131+ ## the static schema -- it comes from GLEAN_EMBED_DIMENSION.
132+ g.db.exec &"""CREATE VIRTUAL TABLE IF NOT EXISTS recs.feed_embeddings
133+ USING vec0(feed_url TEXT PRIMARY KEY, embedding float[{dimension}])"""
134+ g.db.exec &"""CREATE VIRTUAL TABLE IF NOT EXISTS recs.article_embeddings
135+ USING vec0(article_id INTEGER PRIMARY KEY, embedding float[{dimension}])"""
136+
137+proc close*(g: var GleanDb) =
138+ g.db.close()
139+
140+proc journalMode*(g: GleanDb, schemaName = ""): string =
141+ let sql = if schemaName.len == 0: "PRAGMA journal_mode"
142+ else: &"PRAGMA {schemaName}.journal_mode"
143+ g.db.queryText(sql).get("")
144+
145+proc attachedSchemas*(g: GleanDb): seq[string] =
146+ for row in g.db.rows("PRAGMA database_list"):
147+ result.add row[1]
added spike/nim/atproto/schema.nim +232 -0
new file mode 100644
@@ -0,0 +1,232 @@
1+## Glean's schema, in the three databases it uses.
2+##
3+## Extracted mechanically from internal/db/db.go rather than retyped: 44 DDL
4+## statements transcribed by hand would drift from the Go definition, and a
5+## column that differs by a default or a CHECK is the kind of thing that only
6+## shows up as a constraint violation months later.
7+##
8+## Regenerate with tools/extract_schema.py after changing the Go schema.
9+
10+const usersSchema* = [
11+ """CREATE TABLE IF NOT EXISTS users (
12+ did TEXT PRIMARY KEY,
13+ indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
14+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
15+ follows_dirty BOOLEAN NOT NULL DEFAULT 1
16+ )""",
17+ """CREATE TABLE IF NOT EXISTS follows (
18+ user_did TEXT NOT NULL,
19+ target_did TEXT NOT NULL,
20+ uri TEXT,
21+ cid TEXT,
22+ followed_at DATETIME,
23+ PRIMARY KEY (user_did, target_did)
24+ )""",
25+ """CREATE TABLE IF NOT EXISTS oauth_auth_requests (
26+ state TEXT PRIMARY KEY,
27+ data TEXT NOT NULL
28+ )""",
29+ """CREATE TABLE IF NOT EXISTS oauth_sessions (
30+ account_did TEXT NOT NULL,
31+ session_id TEXT NOT NULL,
32+ data TEXT NOT NULL,
33+ PRIMARY KEY (account_did, session_id)
34+ )""",
35+ """CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)""",
36+ """CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)""",
37+ """CREATE TABLE IF NOT EXISTS user_settings (
38+ did TEXT PRIMARY KEY,
39+ languages TEXT,
40+ expanded_view BOOLEAN NOT NULL DEFAULT 0,
41+ digest_enabled BOOLEAN NOT NULL DEFAULT 0,
42+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
43+ )""",
44+ """CREATE TABLE IF NOT EXISTS dismissed_recommendations (
45+ user_did TEXT NOT NULL,
46+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article', 'person')),
47+ target_id TEXT NOT NULL,
48+ reason TEXT,
49+ dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
50+ PRIMARY KEY (user_did, target_type, target_id)
51+ )""",
52+ """CREATE TABLE IF NOT EXISTS recommendation_impressions (
53+ user_did TEXT NOT NULL,
54+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article', 'person')),
55+ target_id TEXT NOT NULL,
56+ first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
57+ last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
58+ shown_count INTEGER NOT NULL DEFAULT 1,
59+ acted BOOLEAN NOT NULL DEFAULT 0,
60+ PRIMARY KEY (user_did, target_type, target_id)
61+ )""",
62+ """CREATE INDEX IF NOT EXISTS idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)""",
63+ """CREATE TABLE IF NOT EXISTS jetstream_cursor (
64+ id INTEGER PRIMARY KEY CHECK(id = 1),
65+ cursor_us INTEGER NOT NULL
66+ )""",
67+]
68+
69+const articlesSchema* = [
70+ """CREATE TABLE IF NOT EXISTS articles.feeds (
71+ feed_url TEXT PRIMARY KEY,
72+ title TEXT,
73+ site_url TEXT,
74+ description TEXT,
75+ feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json', 'atproto')),
76+ last_fetched_at DATETIME,
77+ last_error TEXT,
78+ subscriber_count INTEGER NOT NULL DEFAULT 0,
79+ consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
80+ error_count INTEGER NOT NULL DEFAULT 0,
81+ favicon_url TEXT
82+ )""",
83+ """CREATE TABLE IF NOT EXISTS articles.subscriptions (
84+ id INTEGER PRIMARY KEY AUTOINCREMENT,
85+ user_did TEXT NOT NULL,
86+ feed_url TEXT NOT NULL,
87+ title TEXT,
88+ category TEXT,
89+ added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
90+ uri TEXT,
91+ cid TEXT,
92+ UNIQUE(user_did, feed_url)
93+ )""",
94+ """CREATE TABLE IF NOT EXISTS articles.articles (
95+ id INTEGER PRIMARY KEY AUTOINCREMENT,
96+ feed_url TEXT NOT NULL,
97+ guid TEXT NOT NULL,
98+ title TEXT NOT NULL DEFAULT '',
99+ url TEXT,
100+ author TEXT,
101+ summary TEXT,
102+ content TEXT,
103+ full_content TEXT,
104+ published DATETIME,
105+ updated DATETIME,
106+ fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
107+ language TEXT NOT NULL DEFAULT '',
108+ UNIQUE(feed_url, guid)
109+ )""",
110+ """CREATE TABLE IF NOT EXISTS articles.read_state (
111+ user_did TEXT NOT NULL,
112+ article_id INTEGER NOT NULL,
113+ is_read BOOLEAN NOT NULL DEFAULT 0,
114+ read_at DATETIME,
115+ PRIMARY KEY (user_did, article_id)
116+ )""",
117+ """CREATE TABLE IF NOT EXISTS articles.read_state_history (
118+ user_did TEXT NOT NULL,
119+ feed_url TEXT NOT NULL,
120+ guid TEXT NOT NULL,
121+ is_read BOOLEAN NOT NULL DEFAULT 0,
122+ read_at DATETIME,
123+ PRIMARY KEY (user_did, feed_url, guid)
124+ )""",
125+ """CREATE TABLE IF NOT EXISTS articles.annotations (
126+ id INTEGER PRIMARY KEY AUTOINCREMENT,
127+ uri TEXT NOT NULL UNIQUE,
128+ author_did TEXT NOT NULL,
129+ feed_url TEXT NOT NULL,
130+ article_url TEXT NOT NULL,
131+ quote TEXT,
132+ note TEXT,
133+ tags TEXT,
134+ rating INTEGER,
135+ created_at DATETIME NOT NULL,
136+ cid TEXT
137+ )""",
138+ """CREATE TABLE IF NOT EXISTS articles.likes (
139+ id INTEGER PRIMARY KEY AUTOINCREMENT,
140+ uri TEXT NOT NULL UNIQUE,
141+ author_did TEXT NOT NULL,
142+ feed_url TEXT NOT NULL,
143+ article_url TEXT NOT NULL,
144+ created_at DATETIME NOT NULL,
145+ cid TEXT,
146+ UNIQUE(author_did, feed_url, article_url)
147+ )""",
148+ """CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)""",
149+ """CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)""",
150+ """CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)""",
151+ """CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)""",
152+ """CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0""",
153+ """CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)""",
154+ """CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)""",
155+ """CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)""",
156+ """CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)""",
157+ """CREATE INDEX IF NOT EXISTS articles.idx_articles_language ON articles(language)""",
158+ """CREATE INDEX IF NOT EXISTS articles.idx_likes_article_created ON likes(feed_url, article_url, created_at)""",
159+ """CREATE INDEX IF NOT EXISTS articles.idx_annotations_feed_article_created ON annotations(feed_url, article_url, created_at)""",
160+ """CREATE INDEX IF NOT EXISTS articles.idx_likes_created_author ON likes(created_at, author_did)""",
161+ """CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)""",
162+ """CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN
163+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
164+ END""",
165+ """CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN
166+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
167+ END""",
168+ """CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN
169+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
170+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
171+ END""",
172+]
173+
174+const recsSchema* = [
175+ """CREATE TABLE IF NOT EXISTS recs.feed_similarity (
176+ feed_a TEXT NOT NULL,
177+ feed_b TEXT NOT NULL,
178+ jaccard REAL NOT NULL,
179+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
180+ PRIMARY KEY (feed_a, feed_b),
181+ CHECK(feed_a < feed_b)
182+ )""",
183+ """CREATE TABLE IF NOT EXISTS recs.user_similarity (
184+ user_a TEXT NOT NULL,
185+ user_b TEXT NOT NULL,
186+ jaccard REAL NOT NULL,
187+ common_feeds INTEGER NOT NULL,
188+ common_likes INTEGER NOT NULL DEFAULT 0,
189+ common_tags INTEGER NOT NULL DEFAULT 0,
190+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
191+ PRIMARY KEY (user_a, user_b),
192+ CHECK(user_a < user_b)
193+ )""",
194+ """CREATE TABLE IF NOT EXISTS recs.follow_distances (
195+ user_a TEXT NOT NULL,
196+ user_b TEXT NOT NULL,
197+ distance INTEGER NOT NULL CHECK(distance IN (1, 2, 3)),
198+ PRIMARY KEY (user_a, user_b)
199+ )""",
200+ """CREATE TABLE IF NOT EXISTS recs.user_signal_weights (
201+ user_did TEXT PRIMARY KEY,
202+ w_sub REAL NOT NULL DEFAULT 1.0,
203+ w_like REAL NOT NULL DEFAULT 0.5,
204+ w_tag REAL NOT NULL DEFAULT 0.3,
205+ w_social REAL NOT NULL DEFAULT 0.7,
206+ w_pop REAL NOT NULL DEFAULT 0.2,
207+ w_category REAL NOT NULL DEFAULT 0.4,
208+ w_content REAL NOT NULL DEFAULT 0.4,
209+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
210+ )""",
211+ """CREATE TABLE IF NOT EXISTS recs.user_signal_profiles (
212+ user_did TEXT PRIMARY KEY,
213+ total_likes INTEGER NOT NULL DEFAULT 0,
214+ total_tags INTEGER NOT NULL DEFAULT 0,
215+ top_categories TEXT,
216+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
217+ )""",
218+ """CREATE TABLE IF NOT EXISTS recs.feed_embedding_meta (
219+ feed_url TEXT PRIMARY KEY,
220+ source_text TEXT NOT NULL DEFAULT ''
221+ )""",
222+ """CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)""",
223+ """CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)""",
224+ """CREATE TABLE IF NOT EXISTS recs.precomputed_recommendations (
225+ user_did TEXT NOT NULL,
226+ rec_type TEXT NOT NULL CHECK(rec_type IN ('feed', 'article', 'person')),
227+ data TEXT NOT NULL,
228+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
229+ PRIMARY KEY (user_did, rec_type)
230+ )""",
231+]
232+
new file mode 100644
@@ -0,0 +1,232 @@
1+## Glean's schema, in the three databases it uses.
2+##
3+## Extracted mechanically from internal/db/db.go rather than retyped: 44 DDL
4+## statements transcribed by hand would drift from the Go definition, and a
5+## column that differs by a default or a CHECK is the kind of thing that only
6+## shows up as a constraint violation months later.
7+##
8+## Regenerate with tools/extract_schema.py after changing the Go schema.
9+
10+const usersSchema* = [
11+ """CREATE TABLE IF NOT EXISTS users (
12+ did TEXT PRIMARY KEY,
13+ indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
14+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
15+ follows_dirty BOOLEAN NOT NULL DEFAULT 1
16+ )""",
17+ """CREATE TABLE IF NOT EXISTS follows (
18+ user_did TEXT NOT NULL,
19+ target_did TEXT NOT NULL,
20+ uri TEXT,
21+ cid TEXT,
22+ followed_at DATETIME,
23+ PRIMARY KEY (user_did, target_did)
24+ )""",
25+ """CREATE TABLE IF NOT EXISTS oauth_auth_requests (
26+ state TEXT PRIMARY KEY,
27+ data TEXT NOT NULL
28+ )""",
29+ """CREATE TABLE IF NOT EXISTS oauth_sessions (
30+ account_did TEXT NOT NULL,
31+ session_id TEXT NOT NULL,
32+ data TEXT NOT NULL,
33+ PRIMARY KEY (account_did, session_id)
34+ )""",
35+ """CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)""",
36+ """CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)""",
37+ """CREATE TABLE IF NOT EXISTS user_settings (
38+ did TEXT PRIMARY KEY,
39+ languages TEXT,
40+ expanded_view BOOLEAN NOT NULL DEFAULT 0,
41+ digest_enabled BOOLEAN NOT NULL DEFAULT 0,
42+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
43+ )""",
44+ """CREATE TABLE IF NOT EXISTS dismissed_recommendations (
45+ user_did TEXT NOT NULL,
46+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article', 'person')),
47+ target_id TEXT NOT NULL,
48+ reason TEXT,
49+ dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
50+ PRIMARY KEY (user_did, target_type, target_id)
51+ )""",
52+ """CREATE TABLE IF NOT EXISTS recommendation_impressions (
53+ user_did TEXT NOT NULL,
54+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article', 'person')),
55+ target_id TEXT NOT NULL,
56+ first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
57+ last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
58+ shown_count INTEGER NOT NULL DEFAULT 1,
59+ acted BOOLEAN NOT NULL DEFAULT 0,
60+ PRIMARY KEY (user_did, target_type, target_id)
61+ )""",
62+ """CREATE INDEX IF NOT EXISTS idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)""",
63+ """CREATE TABLE IF NOT EXISTS jetstream_cursor (
64+ id INTEGER PRIMARY KEY CHECK(id = 1),
65+ cursor_us INTEGER NOT NULL
66+ )""",
67+]
68+
69+const articlesSchema* = [
70+ """CREATE TABLE IF NOT EXISTS articles.feeds (
71+ feed_url TEXT PRIMARY KEY,
72+ title TEXT,
73+ site_url TEXT,
74+ description TEXT,
75+ feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json', 'atproto')),
76+ last_fetched_at DATETIME,
77+ last_error TEXT,
78+ subscriber_count INTEGER NOT NULL DEFAULT 0,
79+ consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
80+ error_count INTEGER NOT NULL DEFAULT 0,
81+ favicon_url TEXT
82+ )""",
83+ """CREATE TABLE IF NOT EXISTS articles.subscriptions (
84+ id INTEGER PRIMARY KEY AUTOINCREMENT,
85+ user_did TEXT NOT NULL,
86+ feed_url TEXT NOT NULL,
87+ title TEXT,
88+ category TEXT,
89+ added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
90+ uri TEXT,
91+ cid TEXT,
92+ UNIQUE(user_did, feed_url)
93+ )""",
94+ """CREATE TABLE IF NOT EXISTS articles.articles (
95+ id INTEGER PRIMARY KEY AUTOINCREMENT,
96+ feed_url TEXT NOT NULL,
97+ guid TEXT NOT NULL,
98+ title TEXT NOT NULL DEFAULT '',
99+ url TEXT,
100+ author TEXT,
101+ summary TEXT,
102+ content TEXT,
103+ full_content TEXT,
104+ published DATETIME,
105+ updated DATETIME,
106+ fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
107+ language TEXT NOT NULL DEFAULT '',
108+ UNIQUE(feed_url, guid)
109+ )""",
110+ """CREATE TABLE IF NOT EXISTS articles.read_state (
111+ user_did TEXT NOT NULL,
112+ article_id INTEGER NOT NULL,
113+ is_read BOOLEAN NOT NULL DEFAULT 0,
114+ read_at DATETIME,
115+ PRIMARY KEY (user_did, article_id)
116+ )""",
117+ """CREATE TABLE IF NOT EXISTS articles.read_state_history (
118+ user_did TEXT NOT NULL,
119+ feed_url TEXT NOT NULL,
120+ guid TEXT NOT NULL,
121+ is_read BOOLEAN NOT NULL DEFAULT 0,
122+ read_at DATETIME,
123+ PRIMARY KEY (user_did, feed_url, guid)
124+ )""",
125+ """CREATE TABLE IF NOT EXISTS articles.annotations (
126+ id INTEGER PRIMARY KEY AUTOINCREMENT,
127+ uri TEXT NOT NULL UNIQUE,
128+ author_did TEXT NOT NULL,
129+ feed_url TEXT NOT NULL,
130+ article_url TEXT NOT NULL,
131+ quote TEXT,
132+ note TEXT,
133+ tags TEXT,
134+ rating INTEGER,
135+ created_at DATETIME NOT NULL,
136+ cid TEXT
137+ )""",
138+ """CREATE TABLE IF NOT EXISTS articles.likes (
139+ id INTEGER PRIMARY KEY AUTOINCREMENT,
140+ uri TEXT NOT NULL UNIQUE,
141+ author_did TEXT NOT NULL,
142+ feed_url TEXT NOT NULL,
143+ article_url TEXT NOT NULL,
144+ created_at DATETIME NOT NULL,
145+ cid TEXT,
146+ UNIQUE(author_did, feed_url, article_url)
147+ )""",
148+ """CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)""",
149+ """CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)""",
150+ """CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)""",
151+ """CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)""",
152+ """CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0""",
153+ """CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)""",
154+ """CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)""",
155+ """CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)""",
156+ """CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)""",
157+ """CREATE INDEX IF NOT EXISTS articles.idx_articles_language ON articles(language)""",
158+ """CREATE INDEX IF NOT EXISTS articles.idx_likes_article_created ON likes(feed_url, article_url, created_at)""",
159+ """CREATE INDEX IF NOT EXISTS articles.idx_annotations_feed_article_created ON annotations(feed_url, article_url, created_at)""",
160+ """CREATE INDEX IF NOT EXISTS articles.idx_likes_created_author ON likes(created_at, author_did)""",
161+ """CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)""",
162+ """CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN
163+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
164+ END""",
165+ """CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN
166+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
167+ END""",
168+ """CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN
169+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
170+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
171+ END""",
172+]
173+
174+const recsSchema* = [
175+ """CREATE TABLE IF NOT EXISTS recs.feed_similarity (
176+ feed_a TEXT NOT NULL,
177+ feed_b TEXT NOT NULL,
178+ jaccard REAL NOT NULL,
179+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
180+ PRIMARY KEY (feed_a, feed_b),
181+ CHECK(feed_a < feed_b)
182+ )""",
183+ """CREATE TABLE IF NOT EXISTS recs.user_similarity (
184+ user_a TEXT NOT NULL,
185+ user_b TEXT NOT NULL,
186+ jaccard REAL NOT NULL,
187+ common_feeds INTEGER NOT NULL,
188+ common_likes INTEGER NOT NULL DEFAULT 0,
189+ common_tags INTEGER NOT NULL DEFAULT 0,
190+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
191+ PRIMARY KEY (user_a, user_b),
192+ CHECK(user_a < user_b)
193+ )""",
194+ """CREATE TABLE IF NOT EXISTS recs.follow_distances (
195+ user_a TEXT NOT NULL,
196+ user_b TEXT NOT NULL,
197+ distance INTEGER NOT NULL CHECK(distance IN (1, 2, 3)),
198+ PRIMARY KEY (user_a, user_b)
199+ )""",
200+ """CREATE TABLE IF NOT EXISTS recs.user_signal_weights (
201+ user_did TEXT PRIMARY KEY,
202+ w_sub REAL NOT NULL DEFAULT 1.0,
203+ w_like REAL NOT NULL DEFAULT 0.5,
204+ w_tag REAL NOT NULL DEFAULT 0.3,
205+ w_social REAL NOT NULL DEFAULT 0.7,
206+ w_pop REAL NOT NULL DEFAULT 0.2,
207+ w_category REAL NOT NULL DEFAULT 0.4,
208+ w_content REAL NOT NULL DEFAULT 0.4,
209+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
210+ )""",
211+ """CREATE TABLE IF NOT EXISTS recs.user_signal_profiles (
212+ user_did TEXT PRIMARY KEY,
213+ total_likes INTEGER NOT NULL DEFAULT 0,
214+ total_tags INTEGER NOT NULL DEFAULT 0,
215+ top_categories TEXT,
216+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
217+ )""",
218+ """CREATE TABLE IF NOT EXISTS recs.feed_embedding_meta (
219+ feed_url TEXT PRIMARY KEY,
220+ source_text TEXT NOT NULL DEFAULT ''
221+ )""",
222+ """CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)""",
223+ """CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)""",
224+ """CREATE TABLE IF NOT EXISTS recs.precomputed_recommendations (
225+ user_did TEXT NOT NULL,
226+ rec_type TEXT NOT NULL CHECK(rec_type IN ('feed', 'article', 'person')),
227+ data TEXT NOT NULL,
228+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
229+ PRIMARY KEY (user_did, rec_type)
230+ )""",
231+]
232+
added spike/nim/gleandb_probe.nim +161 -0
new file mode 100644
@@ -0,0 +1,161 @@
1+## Checks the three-database layout, the schema, and the behaviour that
2+## depends on both.
3+##
4+## nim c -r -d:ssl gleandb_probe.nim
5+##
6+## Most of this is about things that fail quietly. An ATTACH that did not
7+## happen, a pragma that did not take, an FTS index whose triggers never fire
8+## -- each leaves a database that works for the first query anyone tries and
9+## degrades or diverges later.
10+
11+import std/[options, os, strformat, strutils]
12+import atproto/[gleandb, schema, sqlite]
13+
14+var failures = 0
15+
16+proc report(name: string, ok: bool, detail = "") =
17+ if not ok: inc failures
18+ let label = if ok: "PASS" else: "FAIL"
19+ if detail.len > 0: echo &" {label} {name} -- {detail}"
20+ else: echo &" {label} {name}"
21+
22+proc cleanup(base: string) =
23+ for suffix in ["_users", "_articles", "_recs"]:
24+ for ext in ["", "-wal", "-shm"]:
25+ removeFile(base & suffix & ext)
26+
27+proc main() =
28+ echo "Glean database layer probe"
29+ echo ""
30+
31+ let base = getTempDir() / "glean_db_probe"
32+ cleanup(base)
33+ defer: cleanup(base)
34+
35+ var g = gleandb.open(base)
36+ defer: g.close()
37+
38+ report("schema statements extracted from the Go source",
39+ usersSchema.len + articlesSchema.len + recsSchema.len == 44,
40+ &"{usersSchema.len} + {articlesSchema.len} + {recsSchema.len}")
41+
42+ # --- the three databases ------------------------------------------------
43+ block:
44+ let schemas = g.attachedSchemas()
45+ report("all three databases are attached",
46+ "main" in schemas and "articles" in schemas and "recs" in schemas,
47+ schemas.join(", "))
48+ report("each is a separate file on disk",
49+ fileExists(base & "_users") and fileExists(base & "_articles") and
50+ fileExists(base & "_recs"))
51+
52+ # --- pragmas ------------------------------------------------------------
53+ # WAL has to be set per schema: PRAGMA applies to one at a time, so setting
54+ # it on main leaves the others journalling the slow way.
55+ block:
56+ for name in ["", "articles", "recs"]:
57+ let mode = g.journalMode(name).toLowerAscii
58+ report(&"WAL is on for {(if name.len == 0: \"main\" else: name)}",
59+ mode == "wal", mode)
60+
61+ # --- custom SQL functions ----------------------------------------------
62+ block:
63+ let e = g.db.queryText("SELECT exp(1.0)")
64+ report("exp() is callable from SQL",
65+ e.isSome and e.get.startsWith("2.718"), e.get(""))
66+ let l = g.db.queryText("SELECT log(2.718281828459045)")
67+ report("log() is callable from SQL",
68+ l.isSome and l.get.startsWith("1.0") or l.get("") == "1.0", l.get(""))
69+ # log(0) is -inf; returning NULL keeps an ORDER BY sane.
70+ report("log() of zero is NULL",
71+ g.db.queryText("SELECT log(0)").isNone)
72+ report("log() of a negative is NULL",
73+ g.db.queryText("SELECT log(-5)").isNone)
74+
75+ # --- schema -------------------------------------------------------------
76+ g.migrate()
77+ report("migrate() is idempotent", (g.migrate(); true))
78+
79+ block:
80+ proc tableCount(schemaName: string): int =
81+ let n = g.db.queryInt(
82+ &"SELECT count(*) FROM {schemaName}.sqlite_master WHERE type='table'")
83+ n.get(0).int
84+ report("users tables created", tableCount("main") >= 7, $tableCount("main"))
85+ report("articles tables created", tableCount("articles") >= 7,
86+ $tableCount("articles"))
87+ report("recs tables created", tableCount("recs") >= 7, $tableCount("recs"))
88+
89+ # --- a real cross-database query ---------------------------------------
90+ # The whole point of attaching rather than opening three connections.
91+ block:
92+ # `users` is keyed on the DID alone -- handles are resolved from atproto
93+ # rather than stored, so there is no handle column to join on.
94+ g.db.run("INSERT INTO users (did) VALUES (?)", p("did:plc:probe"))
95+ g.db.run("""INSERT INTO articles.feeds (feed_url, title)
96+ VALUES (?, ?)""", p("https://ex.test/feed"), p("Example"))
97+ g.db.run("""INSERT INTO articles.subscriptions (user_did, feed_url)
98+ VALUES (?, ?)""", p("did:plc:probe"), p("https://ex.test/feed"))
99+
100+ let joined = g.db.queryText("""
101+ SELECT f.title FROM articles.subscriptions s
102+ JOIN users u ON u.did = s.user_did
103+ JOIN articles.feeds f ON f.feed_url = s.feed_url
104+ WHERE u.did = ?""", p("did:plc:probe"))
105+ report("a query joins across attached databases",
106+ joined == some("Example"), joined.get("nothing"))
107+
108+ # --- FTS5 ---------------------------------------------------------------
109+ # The index is kept current by triggers, so inserting into `articles` alone
110+ # must make the row findable. If the triggers were missed, search silently
111+ # returns nothing while everything else looks fine.
112+ block:
113+ g.db.run("""INSERT INTO articles.articles
114+ (feed_url, guid, title, summary, content, author)
115+ VALUES (?, ?, ?, ?, ?, ?)""",
116+ p("https://ex.test/feed"), p("g1"),
117+ p("Nim systems programming"), p("a summary"),
118+ p("the body text"), p("An Author"))
119+
120+ let hit = g.db.queryText(
121+ "SELECT title FROM articles.articles_fts WHERE articles_fts MATCH ?",
122+ p("systems"))
123+ report("FTS5 triggers index new rows", hit.isSome, hit.get("no match"))
124+
125+ g.db.run("UPDATE articles.articles SET title = ? WHERE guid = ?",
126+ p("Rust systems programming"), p("g1"))
127+ let updated = g.db.queryText(
128+ "SELECT title FROM articles.articles_fts WHERE articles_fts MATCH ?",
129+ p("Rust"))
130+ report("FTS5 triggers follow updates", updated.isSome, updated.get("no match"))
131+
132+ g.db.run("DELETE FROM articles.articles WHERE guid = ?", p("g1"))
133+ let afterDelete = g.db.queryText(
134+ "SELECT title FROM articles.articles_fts WHERE articles_fts MATCH ?",
135+ p("systems"))
136+ # The detail is only interesting when this fails, and printing the
137+ # fallback on success reads like the opposite of what happened.
138+ report("FTS5 triggers follow deletes", afterDelete.isNone,
139+ if afterDelete.isNone: "row gone from the index"
140+ else: "STILL INDEXED: " & afterDelete.get)
141+
142+ # --- vec0 ---------------------------------------------------------------
143+ block:
144+ g.createEmbeddingTables(dimension = 4)
145+ report("embedding tables take their dimension from config",
146+ g.db.queryInt(
147+ "SELECT count(*) FROM recs.sqlite_master WHERE name LIKE '%embeddings%'"
148+ ).get(0) > 0)
149+ # Proves sqlite-vec is registered on this connection, not just linked.
150+ let ver = g.db.queryText("SELECT vec_version()")
151+ report("sqlite-vec is live on the glean connection", ver.isSome, ver.get(""))
152+
153+ echo ""
154+ if failures == 0:
155+ echo "RESULT: the database layer matches Glean's layout and behaviour."
156+ else:
157+ echo &"RESULT: {failures} check(s) failed."
158+ quit 1
159+
160+when isMainModule:
161+ main()
new file mode 100644
@@ -0,0 +1,161 @@
1+## Checks the three-database layout, the schema, and the behaviour that
2+## depends on both.
3+##
4+## nim c -r -d:ssl gleandb_probe.nim
5+##
6+## Most of this is about things that fail quietly. An ATTACH that did not
7+## happen, a pragma that did not take, an FTS index whose triggers never fire
8+## -- each leaves a database that works for the first query anyone tries and
9+## degrades or diverges later.
10+
11+import std/[options, os, strformat, strutils]
12+import atproto/[gleandb, schema, sqlite]
13+
14+var failures = 0
15+
16+proc report(name: string, ok: bool, detail = "") =
17+ if not ok: inc failures
18+ let label = if ok: "PASS" else: "FAIL"
19+ if detail.len > 0: echo &" {label} {name} -- {detail}"
20+ else: echo &" {label} {name}"
21+
22+proc cleanup(base: string) =
23+ for suffix in ["_users", "_articles", "_recs"]:
24+ for ext in ["", "-wal", "-shm"]:
25+ removeFile(base & suffix & ext)
26+
27+proc main() =
28+ echo "Glean database layer probe"
29+ echo ""
30+
31+ let base = getTempDir() / "glean_db_probe"
32+ cleanup(base)
33+ defer: cleanup(base)
34+
35+ var g = gleandb.open(base)
36+ defer: g.close()
37+
38+ report("schema statements extracted from the Go source",
39+ usersSchema.len + articlesSchema.len + recsSchema.len == 44,
40+ &"{usersSchema.len} + {articlesSchema.len} + {recsSchema.len}")
41+
42+ # --- the three databases ------------------------------------------------
43+ block:
44+ let schemas = g.attachedSchemas()
45+ report("all three databases are attached",
46+ "main" in schemas and "articles" in schemas and "recs" in schemas,
47+ schemas.join(", "))
48+ report("each is a separate file on disk",
49+ fileExists(base & "_users") and fileExists(base & "_articles") and
50+ fileExists(base & "_recs"))
51+
52+ # --- pragmas ------------------------------------------------------------
53+ # WAL has to be set per schema: PRAGMA applies to one at a time, so setting
54+ # it on main leaves the others journalling the slow way.
55+ block:
56+ for name in ["", "articles", "recs"]:
57+ let mode = g.journalMode(name).toLowerAscii
58+ report(&"WAL is on for {(if name.len == 0: \"main\" else: name)}",
59+ mode == "wal", mode)
60+
61+ # --- custom SQL functions ----------------------------------------------
62+ block:
63+ let e = g.db.queryText("SELECT exp(1.0)")
64+ report("exp() is callable from SQL",
65+ e.isSome and e.get.startsWith("2.718"), e.get(""))
66+ let l = g.db.queryText("SELECT log(2.718281828459045)")
67+ report("log() is callable from SQL",
68+ l.isSome and l.get.startsWith("1.0") or l.get("") == "1.0", l.get(""))
69+ # log(0) is -inf; returning NULL keeps an ORDER BY sane.
70+ report("log() of zero is NULL",
71+ g.db.queryText("SELECT log(0)").isNone)
72+ report("log() of a negative is NULL",
73+ g.db.queryText("SELECT log(-5)").isNone)
74+
75+ # --- schema -------------------------------------------------------------
76+ g.migrate()
77+ report("migrate() is idempotent", (g.migrate(); true))
78+
79+ block:
80+ proc tableCount(schemaName: string): int =
81+ let n = g.db.queryInt(
82+ &"SELECT count(*) FROM {schemaName}.sqlite_master WHERE type='table'")
83+ n.get(0).int
84+ report("users tables created", tableCount("main") >= 7, $tableCount("main"))
85+ report("articles tables created", tableCount("articles") >= 7,
86+ $tableCount("articles"))
87+ report("recs tables created", tableCount("recs") >= 7, $tableCount("recs"))
88+
89+ # --- a real cross-database query ---------------------------------------
90+ # The whole point of attaching rather than opening three connections.
91+ block:
92+ # `users` is keyed on the DID alone -- handles are resolved from atproto
93+ # rather than stored, so there is no handle column to join on.
94+ g.db.run("INSERT INTO users (did) VALUES (?)", p("did:plc:probe"))
95+ g.db.run("""INSERT INTO articles.feeds (feed_url, title)
96+ VALUES (?, ?)""", p("https://ex.test/feed"), p("Example"))
97+ g.db.run("""INSERT INTO articles.subscriptions (user_did, feed_url)
98+ VALUES (?, ?)""", p("did:plc:probe"), p("https://ex.test/feed"))
99+
100+ let joined = g.db.queryText("""
101+ SELECT f.title FROM articles.subscriptions s
102+ JOIN users u ON u.did = s.user_did
103+ JOIN articles.feeds f ON f.feed_url = s.feed_url
104+ WHERE u.did = ?""", p("did:plc:probe"))
105+ report("a query joins across attached databases",
106+ joined == some("Example"), joined.get("nothing"))
107+
108+ # --- FTS5 ---------------------------------------------------------------
109+ # The index is kept current by triggers, so inserting into `articles` alone
110+ # must make the row findable. If the triggers were missed, search silently
111+ # returns nothing while everything else looks fine.
112+ block:
113+ g.db.run("""INSERT INTO articles.articles
114+ (feed_url, guid, title, summary, content, author)
115+ VALUES (?, ?, ?, ?, ?, ?)""",
116+ p("https://ex.test/feed"), p("g1"),
117+ p("Nim systems programming"), p("a summary"),
118+ p("the body text"), p("An Author"))
119+
120+ let hit = g.db.queryText(
121+ "SELECT title FROM articles.articles_fts WHERE articles_fts MATCH ?",
122+ p("systems"))
123+ report("FTS5 triggers index new rows", hit.isSome, hit.get("no match"))
124+
125+ g.db.run("UPDATE articles.articles SET title = ? WHERE guid = ?",
126+ p("Rust systems programming"), p("g1"))
127+ let updated = g.db.queryText(
128+ "SELECT title FROM articles.articles_fts WHERE articles_fts MATCH ?",
129+ p("Rust"))
130+ report("FTS5 triggers follow updates", updated.isSome, updated.get("no match"))
131+
132+ g.db.run("DELETE FROM articles.articles WHERE guid = ?", p("g1"))
133+ let afterDelete = g.db.queryText(
134+ "SELECT title FROM articles.articles_fts WHERE articles_fts MATCH ?",
135+ p("systems"))
136+ # The detail is only interesting when this fails, and printing the
137+ # fallback on success reads like the opposite of what happened.
138+ report("FTS5 triggers follow deletes", afterDelete.isNone,
139+ if afterDelete.isNone: "row gone from the index"
140+ else: "STILL INDEXED: " & afterDelete.get)
141+
142+ # --- vec0 ---------------------------------------------------------------
143+ block:
144+ g.createEmbeddingTables(dimension = 4)
145+ report("embedding tables take their dimension from config",
146+ g.db.queryInt(
147+ "SELECT count(*) FROM recs.sqlite_master WHERE name LIKE '%embeddings%'"
148+ ).get(0) > 0)
149+ # Proves sqlite-vec is registered on this connection, not just linked.
150+ let ver = g.db.queryText("SELECT vec_version()")
151+ report("sqlite-vec is live on the glean connection", ver.isSome, ver.get(""))
152+
153+ echo ""
154+ if failures == 0:
155+ echo "RESULT: the database layer matches Glean's layout and behaviour."
156+ else:
157+ echo &"RESULT: {failures} check(s) failed."
158+ quit 1
159+
160+when isMainModule:
161+ main()
modified spike/nim/readme.md +41 -0
@@ -16,6 +16,7 @@ nim c -r -d:ssl xrpc_probe.nim # signing
1616 nim c -r -d:ssl jetstream_probe.nim # firehose
1717 nim c -r -d:ssl reconnect_probe.nim # supervisor
1818 nim c -r -d:ssl store_probe.nim # persistence
19+nim c -r -d:ssl gleandb_probe.nim # database
1920 ```
2021
2122 Reusable code lives in `atproto/`; the `*_probe.nim` files are the checks.
@@ -244,6 +245,46 @@ one is a restart that silently drops every session.
244245 refresh token; together they are the account. The database file wants the
245246 protection of a password store, not of a cache.
246247
248+## 8. The database layer — foundation done
249+
250+`atproto/gleandb.nim` opens Glean's storage the way the Go backend does:
251+`<base>_users` as `main`, with `_articles` and `_recs` **attached**, so a
252+query can join across them while each file still checkpoints and vacuums
253+alone — and so the derived recommendation tables can be dropped and rebuilt
254+without touching anything a user wrote. The suffixes match, so a database
255+written by either implementation is readable by the other.
256+
257+Pragmas are repeated per schema rather than set once, which is the detail
258+worth knowing: `PRAGMA` applies to one database at a time, so setting WAL on
259+`main` leaves `articles` journalling the slow way. The probe asserts WAL on
260+all three rather than trusting the call.
261+
262+`exp()` and `log()` are registered as SQL functions, as the Go connection
263+does. Nothing in the current SQL calls either; they exist so a scoring query
264+can decay a weight without pulling rows into the application. `log()` returns
265+NULL for zero and negatives, since `-inf` and NaN both poison an `ORDER BY`.
266+
267+**The schema is extracted, not retyped.** `tools/extract_schema.py` pulls all
268+44 DDL statements out of `internal/db/db.go` into `atproto/schema.nim`.
269+Hand-transcribing them would drift, and a column differing by a default or a
270+CHECK constraint surfaces months later as a violation nobody can place.
271+
272+The probe leans on things that fail quietly: an ATTACH that did not happen, a
273+pragma that did not take, FTS5 triggers that never fire. That last one is the
274+sharpest — the index is maintained by triggers, so if they were missed,
275+search returns nothing while every other query looks healthy. Insert, update
276+and delete are each checked against the index.
277+
278+### Scope
279+
280+This is the foundation, not the whole layer. Glean's `internal/db` is ~2,800
281+lines of non-test code; what is ported here is the connection, the schema and
282+the primitives, plus a cross-database join, FTS5 and vec0 proven end to end.
283+
284+The remaining query surface — `article.go`, `feed.go`, `social.go`,
285+`retention.go` and friends — is mechanical against this base, and large. It
286+is the first part of the port where the work is volume rather than risk.
287+
247288 ## What this does and does not prove
248289
249290 Proven: the cryptography, the storage layer, and the OAuth flow up to user
@@ -16,6 +16,7 @@ nim c -r -d:ssl xrpc_probe.nim # signing
16 nim c -r -d:ssl jetstream_probe.nim # firehose16 nim c -r -d:ssl jetstream_probe.nim # firehose
17 nim c -r -d:ssl reconnect_probe.nim # supervisor17 nim c -r -d:ssl reconnect_probe.nim # supervisor
18 nim c -r -d:ssl store_probe.nim # persistence18 nim c -r -d:ssl store_probe.nim # persistence
19+nim c -r -d:ssl gleandb_probe.nim # database
19 ```20 ```
20 21
21 Reusable code lives in `atproto/`; the `*_probe.nim` files are the checks.22 Reusable code lives in `atproto/`; the `*_probe.nim` files are the checks.
@@ -244,6 +245,46 @@ one is a restart that silently drops every session.
244 refresh token; together they are the account. The database file wants the245 refresh token; together they are the account. The database file wants the
245 protection of a password store, not of a cache.246 protection of a password store, not of a cache.
246 247
248+## 8. The database layer — foundation done
249+
250+`atproto/gleandb.nim` opens Glean's storage the way the Go backend does:
251+`<base>_users` as `main`, with `_articles` and `_recs` **attached**, so a
252+query can join across them while each file still checkpoints and vacuums
253+alone — and so the derived recommendation tables can be dropped and rebuilt
254+without touching anything a user wrote. The suffixes match, so a database
255+written by either implementation is readable by the other.
256+
257+Pragmas are repeated per schema rather than set once, which is the detail
258+worth knowing: `PRAGMA` applies to one database at a time, so setting WAL on
259+`main` leaves `articles` journalling the slow way. The probe asserts WAL on
260+all three rather than trusting the call.
261+
262+`exp()` and `log()` are registered as SQL functions, as the Go connection
263+does. Nothing in the current SQL calls either; they exist so a scoring query
264+can decay a weight without pulling rows into the application. `log()` returns
265+NULL for zero and negatives, since `-inf` and NaN both poison an `ORDER BY`.
266+
267+**The schema is extracted, not retyped.** `tools/extract_schema.py` pulls all
268+44 DDL statements out of `internal/db/db.go` into `atproto/schema.nim`.
269+Hand-transcribing them would drift, and a column differing by a default or a
270+CHECK constraint surfaces months later as a violation nobody can place.
271+
272+The probe leans on things that fail quietly: an ATTACH that did not happen, a
273+pragma that did not take, FTS5 triggers that never fire. That last one is the
274+sharpest — the index is maintained by triggers, so if they were missed,
275+search returns nothing while every other query looks healthy. Insert, update
276+and delete are each checked against the index.
277+
278+### Scope
279+
280+This is the foundation, not the whole layer. Glean's `internal/db` is ~2,800
281+lines of non-test code; what is ported here is the connection, the schema and
282+the primitives, plus a cross-database join, FTS5 and vec0 proven end to end.
283+
284+The remaining query surface — `article.go`, `feed.go`, `social.go`,
285+`retention.go` and friends — is mechanical against this base, and large. It
286+is the first part of the port where the work is volume rather than risk.
287+
247 ## What this does and does not prove288 ## What this does and does not prove
248 289
249 Proven: the cryptography, the storage layer, and the OAuth flow up to user290 Proven: the cryptography, the storage layer, and the OAuth flow up to user
added spike/nim/tools/extract_schema.py +72 -0
new file mode 100755
@@ -0,0 +1,72 @@
1+#!/usr/bin/env python3
2+"""Regenerate atproto/schema.nim from the Go schema.
3+
4+The DDL lives in internal/db/db.go as three []string literals. Retyping 44
5+statements into Nim would drift -- a differing default or CHECK constraint
6+surfaces months later as a violation nobody can place -- so they are copied
7+mechanically instead.
8+
9+ python3 tools/extract_schema.py
10+"""
11+import pathlib
12+import re
13+import sys
14+
15+GO = pathlib.Path(__file__).resolve().parents[3] / "internal" / "db" / "db.go"
16+OUT = pathlib.Path(__file__).resolve().parents[1] / "atproto" / "schema.nim"
17+
18+HEADER = '''## Glean's schema, in the three databases it uses.
19+##
20+## Extracted mechanically from internal/db/db.go rather than retyped: 44 DDL
21+## statements transcribed by hand would drift from the Go definition, and a
22+## column that differs by a default or a CHECK is the kind of thing that only
23+## shows up as a constraint violation months later.
24+##
25+## Regenerate with tools/extract_schema.py after changing the Go schema.
26+
27+'''
28+
29+
30+def groups(src):
31+ """Locate each `var <name>Schema = []string{ ... }` block by line."""
32+ lines = src.splitlines()
33+ found = []
34+ for i, line in enumerate(lines):
35+ m = re.match(r"var (\w+Schema) = \[\]string\{", line)
36+ if not m:
37+ continue
38+ for j in range(i + 1, len(lines)):
39+ if lines[j] == "}":
40+ found.append((m.group(1), i + 1, j))
41+ break
42+ return lines, found
43+
44+
45+def main():
46+ if not GO.exists():
47+ sys.exit(f"cannot find {GO}")
48+ src = GO.read_text()
49+ lines, found = groups(src)
50+ if not found:
51+ sys.exit("no schema blocks found; did db.go change shape?")
52+
53+ out = [HEADER]
54+ total = 0
55+ for name, start, end in found:
56+ body = "\n".join(lines[start:end])
57+ stmts = re.findall(r"`([^`]*)`", body)
58+ total += len(stmts)
59+ out.append(f"const {name}* = [\n")
60+ for st in stmts:
61+ if '"""' in st:
62+ sys.exit(f"statement contains a triple quote: {st[:60]}")
63+ out.append(f' """{st}""",\n')
64+ out.append("]\n\n")
65+ print(f"{name}: {len(stmts)} statements")
66+
67+ OUT.write_text("".join(out))
68+ print(f"wrote {OUT} ({total} statements)")
69+
70+
71+if __name__ == "__main__":
72+ main()
new file mode 100755
@@ -0,0 +1,72 @@
1+#!/usr/bin/env python3
2+"""Regenerate atproto/schema.nim from the Go schema.
3+
4+The DDL lives in internal/db/db.go as three []string literals. Retyping 44
5+statements into Nim would drift -- a differing default or CHECK constraint
6+surfaces months later as a violation nobody can place -- so they are copied
7+mechanically instead.
8+
9+ python3 tools/extract_schema.py
10+"""
11+import pathlib
12+import re
13+import sys
14+
15+GO = pathlib.Path(__file__).resolve().parents[3] / "internal" / "db" / "db.go"
16+OUT = pathlib.Path(__file__).resolve().parents[1] / "atproto" / "schema.nim"
17+
18+HEADER = '''## Glean's schema, in the three databases it uses.
19+##
20+## Extracted mechanically from internal/db/db.go rather than retyped: 44 DDL
21+## statements transcribed by hand would drift from the Go definition, and a
22+## column that differs by a default or a CHECK is the kind of thing that only
23+## shows up as a constraint violation months later.
24+##
25+## Regenerate with tools/extract_schema.py after changing the Go schema.
26+
27+'''
28+
29+
30+def groups(src):
31+ """Locate each `var <name>Schema = []string{ ... }` block by line."""
32+ lines = src.splitlines()
33+ found = []
34+ for i, line in enumerate(lines):
35+ m = re.match(r"var (\w+Schema) = \[\]string\{", line)
36+ if not m:
37+ continue
38+ for j in range(i + 1, len(lines)):
39+ if lines[j] == "}":
40+ found.append((m.group(1), i + 1, j))
41+ break
42+ return lines, found
43+
44+
45+def main():
46+ if not GO.exists():
47+ sys.exit(f"cannot find {GO}")
48+ src = GO.read_text()
49+ lines, found = groups(src)
50+ if not found:
51+ sys.exit("no schema blocks found; did db.go change shape?")
52+
53+ out = [HEADER]
54+ total = 0
55+ for name, start, end in found:
56+ body = "\n".join(lines[start:end])
57+ stmts = re.findall(r"`([^`]*)`", body)
58+ total += len(stmts)
59+ out.append(f"const {name}* = [\n")
60+ for st in stmts:
61+ if '"""' in st:
62+ sys.exit(f"statement contains a triple quote: {st[:60]}")
63+ out.append(f' """{st}""",\n')
64+ out.append("]\n\n")
65+ print(f"{name}: {len(stmts)} statements")
66+
67+ OUT.write_text("".join(out))
68+ print(f"wrote {OUT} ({total} statements)")
69+
70+
71+if __name__ == "__main__":
72+ main()