Linting + Simplify queryUnverified
6b9f76a parent: 12d6fac modified
internal/atproto/jetstream.go +1 -1 | @@ -56,7 +56,7 @@ func (s *jetstreamScheduler) AddWork(ctx context.Context, _ string, evt *models. | ||
| 56 | 56 | RKey: c.RKey, |
| 57 | 57 | URI: fmt.Sprintf("at://%s/%s/%s", evt.Did, c.Collection, c.RKey), |
| 58 | 58 | CID: c.CID, |
| 59 | - Value: json.RawMessage(c.Record), | |
| 59 | + Value: c.Record, | |
| 60 | 60 | } |
| 61 | 61 | |
| 62 | 62 | if err := s.handler(ctx, e); err != nil { |
| @@ -56,7 +56,7 @@ func (s *jetstreamScheduler) AddWork(ctx context.Context, _ string, evt *models. | |||
| 56 | RKey: c.RKey, | 56 | RKey: c.RKey, |
| 57 | URI: fmt.Sprintf("at://%s/%s/%s", evt.Did, c.Collection, c.RKey), | 57 | URI: fmt.Sprintf("at://%s/%s/%s", evt.Did, c.Collection, c.RKey), |
| 58 | CID: c.CID, | 58 | CID: c.CID, |
| 59 | - Value: json.RawMessage(c.Record), | 59 | + Value: c.Record, |
| 60 | } | 60 | } |
| 61 | 61 | ||
| 62 | if err := s.handler(ctx, e); err != nil { | 62 | if err := s.handler(ctx, e); err != nil { |
modified
internal/atproto/stream_handler.go +18 -12 | @@ -11,6 +11,12 @@ import ( | ||
| 11 | 11 | "pkg.rbrt.fr/glean/internal/db" |
| 12 | 12 | ) |
| 13 | 13 | |
| 14 | +const ( | |
| 15 | + actionCreate = "create" | |
| 16 | + actionUpdate = "update" | |
| 17 | + actionDelete = "delete" | |
| 18 | +) | |
| 19 | + | |
| 14 | 20 | var sentinelErrors = []error{db.ErrDuplicateSubscription, db.ErrDuplicateLike} |
| 15 | 21 | |
| 16 | 22 | func isSentinel(err error) bool { |
| @@ -52,7 +58,7 @@ func (h *StreamDBHandler) Handle(ctx context.Context, event *Event) error { | ||
| 52 | 58 | |
| 53 | 59 | func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) error { |
| 54 | 60 | switch event.Type { |
| 55 | - case "create", "update": | |
| 61 | + case actionCreate, actionUpdate: | |
| 56 | 62 | var rec SubscriptionRecord |
| 57 | 63 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 58 | 64 | return err |
| @@ -68,7 +74,7 @@ func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) | ||
| 68 | 74 | } |
| 69 | 75 | return err |
| 70 | 76 | |
| 71 | - case "delete": | |
| 77 | + case actionDelete: | |
| 72 | 78 | parsed, ok := ParseRecordURI(event.URI) |
| 73 | 79 | if !ok { |
| 74 | 80 | return nil |
| @@ -84,7 +90,7 @@ func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) | ||
| 84 | 90 | |
| 85 | 91 | func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error { |
| 86 | 92 | switch event.Type { |
| 87 | - case "create": | |
| 93 | + case actionCreate: | |
| 88 | 94 | var rec LikeRecord |
| 89 | 95 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 90 | 96 | return err |
| @@ -107,7 +113,7 @@ func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error { | ||
| 107 | 113 | } |
| 108 | 114 | return err |
| 109 | 115 | |
| 110 | - case "delete": | |
| 116 | + case actionDelete: | |
| 111 | 117 | return h.articles.DeleteLike(ctx, event.URI) |
| 112 | 118 | } |
| 113 | 119 | return nil |
| @@ -115,7 +121,7 @@ func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error { | ||
| 115 | 121 | |
| 116 | 122 | func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) error { |
| 117 | 123 | switch event.Type { |
| 118 | - case "create": | |
| 124 | + case actionCreate: | |
| 119 | 125 | var rec AnnotationRecord |
| 120 | 126 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 121 | 127 | return err |
| @@ -141,7 +147,7 @@ func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) er | ||
| 141 | 147 | } |
| 142 | 148 | return h.articles.CreateAnnotation(ctx, a) |
| 143 | 149 | |
| 144 | - case "delete": | |
| 150 | + case actionDelete: | |
| 145 | 151 | return h.articles.DeleteAnnotation(ctx, event.URI) |
| 146 | 152 | } |
| 147 | 153 | return nil |
| @@ -149,7 +155,7 @@ func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) er | ||
| 149 | 155 | |
| 150 | 156 | func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error { |
| 151 | 157 | switch event.Type { |
| 152 | - case "create": | |
| 158 | + case actionCreate: | |
| 153 | 159 | var rec FollowRecord |
| 154 | 160 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 155 | 161 | return err |
| @@ -159,7 +165,7 @@ func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error | ||
| 159 | 165 | } |
| 160 | 166 | return h.users.UpsertFollow(ctx, event.DID, rec.Subject, event.URI, event.CID) |
| 161 | 167 | |
| 162 | - case "delete": | |
| 168 | + case actionDelete: | |
| 163 | 169 | return h.users.DeleteFollowByURI(ctx, event.URI) |
| 164 | 170 | } |
| 165 | 171 | return nil |
| @@ -167,7 +173,7 @@ func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error | ||
| 167 | 173 | |
| 168 | 174 | func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) error { |
| 169 | 175 | switch event.Type { |
| 170 | - case "create", "update": | |
| 176 | + case actionCreate, actionUpdate: | |
| 171 | 177 | var rec MarginNoteRecord |
| 172 | 178 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 173 | 179 | return err |
| @@ -194,14 +200,14 @@ func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) er | ||
| 194 | 200 | } |
| 195 | 201 | return h.articles.CreateAnnotation(ctx, a) |
| 196 | 202 | |
| 197 | - case "delete": | |
| 203 | + case actionDelete: | |
| 198 | 204 | } |
| 199 | 205 | return nil |
| 200 | 206 | } |
| 201 | 207 | |
| 202 | 208 | func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event *Event) error { |
| 203 | 209 | switch event.Type { |
| 204 | - case "create", "update": | |
| 210 | + case actionCreate, actionUpdate: | |
| 205 | 211 | var rec SkyreaderSubscriptionRecord |
| 206 | 212 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 207 | 213 | return err |
| @@ -217,7 +223,7 @@ func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event | ||
| 217 | 223 | } |
| 218 | 224 | return err |
| 219 | 225 | |
| 220 | - case "delete": | |
| 226 | + case actionDelete: | |
| 221 | 227 | } |
| 222 | 228 | return nil |
| 223 | 229 | } |
| @@ -11,6 +11,12 @@ import ( | |||
| 11 | "pkg.rbrt.fr/glean/internal/db" | 11 | "pkg.rbrt.fr/glean/internal/db" |
| 12 | ) | 12 | ) |
| 13 | 13 | ||
| 14 | +const ( | ||
| 15 | + actionCreate = "create" | ||
| 16 | + actionUpdate = "update" | ||
| 17 | + actionDelete = "delete" | ||
| 18 | +) | ||
| 19 | + | ||
| 14 | var sentinelErrors = []error{db.ErrDuplicateSubscription, db.ErrDuplicateLike} | 20 | var sentinelErrors = []error{db.ErrDuplicateSubscription, db.ErrDuplicateLike} |
| 15 | 21 | ||
| 16 | func isSentinel(err error) bool { | 22 | func isSentinel(err error) bool { |
| @@ -52,7 +58,7 @@ func (h *StreamDBHandler) Handle(ctx context.Context, event *Event) error { | |||
| 52 | 58 | ||
| 53 | func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) error { | 59 | func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) error { |
| 54 | switch event.Type { | 60 | switch event.Type { |
| 55 | - case "create", "update": | 61 | + case actionCreate, actionUpdate: |
| 56 | var rec SubscriptionRecord | 62 | var rec SubscriptionRecord |
| 57 | if err := json.Unmarshal(event.Value, &rec); err != nil { | 63 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 58 | return err | 64 | return err |
| @@ -68,7 +74,7 @@ func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) | |||
| 68 | } | 74 | } |
| 69 | return err | 75 | return err |
| 70 | 76 | ||
| 71 | - case "delete": | 77 | + case actionDelete: |
| 72 | parsed, ok := ParseRecordURI(event.URI) | 78 | parsed, ok := ParseRecordURI(event.URI) |
| 73 | if !ok { | 79 | if !ok { |
| 74 | return nil | 80 | return nil |
| @@ -84,7 +90,7 @@ func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) | |||
| 84 | 90 | ||
| 85 | func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error { | 91 | func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error { |
| 86 | switch event.Type { | 92 | switch event.Type { |
| 87 | - case "create": | 93 | + case actionCreate: |
| 88 | var rec LikeRecord | 94 | var rec LikeRecord |
| 89 | if err := json.Unmarshal(event.Value, &rec); err != nil { | 95 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 90 | return err | 96 | return err |
| @@ -107,7 +113,7 @@ func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error { | |||
| 107 | } | 113 | } |
| 108 | return err | 114 | return err |
| 109 | 115 | ||
| 110 | - case "delete": | 116 | + case actionDelete: |
| 111 | return h.articles.DeleteLike(ctx, event.URI) | 117 | return h.articles.DeleteLike(ctx, event.URI) |
| 112 | } | 118 | } |
| 113 | return nil | 119 | return nil |
| @@ -115,7 +121,7 @@ func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error { | |||
| 115 | 121 | ||
| 116 | func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) error { | 122 | func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) error { |
| 117 | switch event.Type { | 123 | switch event.Type { |
| 118 | - case "create": | 124 | + case actionCreate: |
| 119 | var rec AnnotationRecord | 125 | var rec AnnotationRecord |
| 120 | if err := json.Unmarshal(event.Value, &rec); err != nil { | 126 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 121 | return err | 127 | return err |
| @@ -141,7 +147,7 @@ func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) er | |||
| 141 | } | 147 | } |
| 142 | return h.articles.CreateAnnotation(ctx, a) | 148 | return h.articles.CreateAnnotation(ctx, a) |
| 143 | 149 | ||
| 144 | - case "delete": | 150 | + case actionDelete: |
| 145 | return h.articles.DeleteAnnotation(ctx, event.URI) | 151 | return h.articles.DeleteAnnotation(ctx, event.URI) |
| 146 | } | 152 | } |
| 147 | return nil | 153 | return nil |
| @@ -149,7 +155,7 @@ func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) er | |||
| 149 | 155 | ||
| 150 | func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error { | 156 | func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error { |
| 151 | switch event.Type { | 157 | switch event.Type { |
| 152 | - case "create": | 158 | + case actionCreate: |
| 153 | var rec FollowRecord | 159 | var rec FollowRecord |
| 154 | if err := json.Unmarshal(event.Value, &rec); err != nil { | 160 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 155 | return err | 161 | return err |
| @@ -159,7 +165,7 @@ func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error | |||
| 159 | } | 165 | } |
| 160 | return h.users.UpsertFollow(ctx, event.DID, rec.Subject, event.URI, event.CID) | 166 | return h.users.UpsertFollow(ctx, event.DID, rec.Subject, event.URI, event.CID) |
| 161 | 167 | ||
| 162 | - case "delete": | 168 | + case actionDelete: |
| 163 | return h.users.DeleteFollowByURI(ctx, event.URI) | 169 | return h.users.DeleteFollowByURI(ctx, event.URI) |
| 164 | } | 170 | } |
| 165 | return nil | 171 | return nil |
| @@ -167,7 +173,7 @@ func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error | |||
| 167 | 173 | ||
| 168 | func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) error { | 174 | func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) error { |
| 169 | switch event.Type { | 175 | switch event.Type { |
| 170 | - case "create", "update": | 176 | + case actionCreate, actionUpdate: |
| 171 | var rec MarginNoteRecord | 177 | var rec MarginNoteRecord |
| 172 | if err := json.Unmarshal(event.Value, &rec); err != nil { | 178 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 173 | return err | 179 | return err |
| @@ -194,14 +200,14 @@ func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) er | |||
| 194 | } | 200 | } |
| 195 | return h.articles.CreateAnnotation(ctx, a) | 201 | return h.articles.CreateAnnotation(ctx, a) |
| 196 | 202 | ||
| 197 | - case "delete": | 203 | + case actionDelete: |
| 198 | } | 204 | } |
| 199 | return nil | 205 | return nil |
| 200 | } | 206 | } |
| 201 | 207 | ||
| 202 | func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event *Event) error { | 208 | func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event *Event) error { |
| 203 | switch event.Type { | 209 | switch event.Type { |
| 204 | - case "create", "update": | 210 | + case actionCreate, actionUpdate: |
| 205 | var rec SkyreaderSubscriptionRecord | 211 | var rec SkyreaderSubscriptionRecord |
| 206 | if err := json.Unmarshal(event.Value, &rec); err != nil { | 212 | if err := json.Unmarshal(event.Value, &rec); err != nil { |
| 207 | return err | 213 | return err |
| @@ -217,7 +223,7 @@ func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event | |||
| 217 | } | 223 | } |
| 218 | return err | 224 | return err |
| 219 | 225 | ||
| 220 | - case "delete": | 226 | + case actionDelete: |
| 221 | } | 227 | } |
| 222 | return nil | 228 | return nil |
| 223 | } | 229 | } |
modified
internal/atproto/xrpc.go +2 -2 | @@ -430,7 +430,7 @@ func (h *XRPCHandler) ListFeedLists(w http.ResponseWriter, r *http.Request) { | ||
| 430 | 430 | var did, feedURL, title string |
| 431 | 431 | var cat sql.NullString |
| 432 | 432 | if err := subRows.Scan(&did, &feedURL, &title, &cat); err != nil { |
| 433 | - subRows.Close() | |
| 433 | + _ = subRows.Close() | |
| 434 | 434 | http.Error(w, err.Error(), http.StatusInternalServerError) |
| 435 | 435 | return |
| 436 | 436 | } |
| @@ -440,7 +440,7 @@ func (h *XRPCHandler) ListFeedLists(w http.ResponseWriter, r *http.Request) { | ||
| 440 | 440 | Category: cat.String, |
| 441 | 441 | }) |
| 442 | 442 | } |
| 443 | - subRows.Close() | |
| 443 | + _ = subRows.Close() | |
| 444 | 444 | } |
| 445 | 445 | |
| 446 | 446 | for _, u := range users { |
| @@ -430,7 +430,7 @@ func (h *XRPCHandler) ListFeedLists(w http.ResponseWriter, r *http.Request) { | |||
| 430 | var did, feedURL, title string | 430 | var did, feedURL, title string |
| 431 | var cat sql.NullString | 431 | var cat sql.NullString |
| 432 | if err := subRows.Scan(&did, &feedURL, &title, &cat); err != nil { | 432 | if err := subRows.Scan(&did, &feedURL, &title, &cat); err != nil { |
| 433 | - subRows.Close() | 433 | + _ = subRows.Close() |
| 434 | http.Error(w, err.Error(), http.StatusInternalServerError) | 434 | http.Error(w, err.Error(), http.StatusInternalServerError) |
| 435 | return | 435 | return |
| 436 | } | 436 | } |
| @@ -440,7 +440,7 @@ func (h *XRPCHandler) ListFeedLists(w http.ResponseWriter, r *http.Request) { | |||
| 440 | Category: cat.String, | 440 | Category: cat.String, |
| 441 | }) | 441 | }) |
| 442 | } | 442 | } |
| 443 | - subRows.Close() | 443 | + _ = subRows.Close() |
| 444 | } | 444 | } |
| 445 | 445 | ||
| 446 | for _, u := range users { | 446 | for _, u := range users { |
modified
internal/db/article.go +6 -5 | @@ -10,6 +10,8 @@ import ( | ||
| 10 | 10 | "pkg.rbrt.fr/glean/internal/feed" |
| 11 | 11 | ) |
| 12 | 12 | |
| 13 | +const articlesOrderBy = ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?` | |
| 14 | + | |
| 13 | 15 | type ArticleStore struct { |
| 14 | 16 | db *DB |
| 15 | 17 | } |
| @@ -142,7 +144,7 @@ func (s *ArticleStore) ListArticles(ctx context.Context, userDID, feedURL string | ||
| 142 | 144 | } |
| 143 | 145 | |
| 144 | 146 | // Future-published articles (e.g., scheduled) sort last |
| 145 | - query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?` | |
| 147 | + query += articlesOrderBy | |
| 146 | 148 | args = append(args, limit, offset) |
| 147 | 149 | |
| 148 | 150 | rows, err := s.db.QueryContext(ctx, query, args...) |
| @@ -206,7 +208,7 @@ func (s *ArticleStore) ListUnreadArticles(ctx context.Context, userDID, feedURL | ||
| 206 | 208 | } |
| 207 | 209 | |
| 208 | 210 | // Future-published articles (e.g., scheduled) sort last |
| 209 | - query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?` | |
| 211 | + query += articlesOrderBy | |
| 210 | 212 | args = append(args, limit, offset) |
| 211 | 213 | |
| 212 | 214 | rows, err := s.db.QueryContext(ctx, query, args...) |
| @@ -270,7 +272,7 @@ func (s *ArticleStore) ListReadArticles(ctx context.Context, userDID, feedURL st | ||
| 270 | 272 | } |
| 271 | 273 | |
| 272 | 274 | // Future-published articles (e.g., scheduled) sort last |
| 273 | - query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?` | |
| 275 | + query += articlesOrderBy | |
| 274 | 276 | args = append(args, limit, offset) |
| 275 | 277 | |
| 276 | 278 | rows, err := s.db.QueryContext(ctx, query, args...) |
| @@ -339,8 +341,7 @@ func (s *ArticleStore) MarkAllSubscribedRead(ctx context.Context, userDID string | ||
| 339 | 341 | func (s *ArticleStore) GetReadState(ctx context.Context, userDID string, articleID int64) (*ReadState, error) { |
| 340 | 342 | rs := &ReadState{} |
| 341 | 343 | err := s.db.QueryRowContext(ctx, ` |
| 342 | - SELECT user_did, article_id, is_read, read_at | |
| 343 | - FROM articles.read_state WHERE user_did = ? AND article_id = ? | |
| 344 | + SELECT * FROM articles.read_state WHERE user_did = ? AND article_id = ? | |
| 344 | 345 | `, userDID, articleID).Scan(&rs.UserDID, &rs.ArticleID, &rs.IsRead, &rs.ReadAt) |
| 345 | 346 | if err == sql.ErrNoRows { |
| 346 | 347 | return &ReadState{UserDID: userDID, ArticleID: articleID}, nil |
| @@ -10,6 +10,8 @@ import ( | |||
| 10 | "pkg.rbrt.fr/glean/internal/feed" | 10 | "pkg.rbrt.fr/glean/internal/feed" |
| 11 | ) | 11 | ) |
| 12 | 12 | ||
| 13 | +const articlesOrderBy = ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?` | ||
| 14 | + | ||
| 13 | type ArticleStore struct { | 15 | type ArticleStore struct { |
| 14 | db *DB | 16 | db *DB |
| 15 | } | 17 | } |
| @@ -142,7 +144,7 @@ func (s *ArticleStore) ListArticles(ctx context.Context, userDID, feedURL string | |||
| 142 | } | 144 | } |
| 143 | 145 | ||
| 144 | // Future-published articles (e.g., scheduled) sort last | 146 | // Future-published articles (e.g., scheduled) sort last |
| 145 | - query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?` | 147 | + query += articlesOrderBy |
| 146 | args = append(args, limit, offset) | 148 | args = append(args, limit, offset) |
| 147 | 149 | ||
| 148 | rows, err := s.db.QueryContext(ctx, query, args...) | 150 | rows, err := s.db.QueryContext(ctx, query, args...) |
| @@ -206,7 +208,7 @@ func (s *ArticleStore) ListUnreadArticles(ctx context.Context, userDID, feedURL | |||
| 206 | } | 208 | } |
| 207 | 209 | ||
| 208 | // Future-published articles (e.g., scheduled) sort last | 210 | // Future-published articles (e.g., scheduled) sort last |
| 209 | - query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?` | 211 | + query += articlesOrderBy |
| 210 | args = append(args, limit, offset) | 212 | args = append(args, limit, offset) |
| 211 | 213 | ||
| 212 | rows, err := s.db.QueryContext(ctx, query, args...) | 214 | rows, err := s.db.QueryContext(ctx, query, args...) |
| @@ -270,7 +272,7 @@ func (s *ArticleStore) ListReadArticles(ctx context.Context, userDID, feedURL st | |||
| 270 | } | 272 | } |
| 271 | 273 | ||
| 272 | // Future-published articles (e.g., scheduled) sort last | 274 | // Future-published articles (e.g., scheduled) sort last |
| 273 | - query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?` | 275 | + query += articlesOrderBy |
| 274 | args = append(args, limit, offset) | 276 | args = append(args, limit, offset) |
| 275 | 277 | ||
| 276 | rows, err := s.db.QueryContext(ctx, query, args...) | 278 | rows, err := s.db.QueryContext(ctx, query, args...) |
| @@ -339,8 +341,7 @@ func (s *ArticleStore) MarkAllSubscribedRead(ctx context.Context, userDID string | |||
| 339 | func (s *ArticleStore) GetReadState(ctx context.Context, userDID string, articleID int64) (*ReadState, error) { | 341 | func (s *ArticleStore) GetReadState(ctx context.Context, userDID string, articleID int64) (*ReadState, error) { |
| 340 | rs := &ReadState{} | 342 | rs := &ReadState{} |
| 341 | err := s.db.QueryRowContext(ctx, ` | 343 | err := s.db.QueryRowContext(ctx, ` |
| 342 | - SELECT user_did, article_id, is_read, read_at | 344 | + SELECT * FROM articles.read_state WHERE user_did = ? AND article_id = ? |
| 343 | - FROM articles.read_state WHERE user_did = ? AND article_id = ? | ||
| 344 | `, userDID, articleID).Scan(&rs.UserDID, &rs.ArticleID, &rs.IsRead, &rs.ReadAt) | 345 | `, userDID, articleID).Scan(&rs.UserDID, &rs.ArticleID, &rs.IsRead, &rs.ReadAt) |
| 345 | if err == sql.ErrNoRows { | 346 | if err == sql.ErrNoRows { |
| 346 | return &ReadState{UserDID: userDID, ArticleID: articleID}, nil | 347 | return &ReadState{UserDID: userDID, ArticleID: articleID}, nil |
modified
internal/db/db.go +319 -11 | @@ -2,8 +2,10 @@ package db | ||
| 2 | 2 | |
| 3 | 3 | import ( |
| 4 | 4 | "database/sql" |
| 5 | + "fmt" | |
| 5 | 6 | "math" |
| 6 | 7 | "strings" |
| 8 | + "sync/atomic" | |
| 7 | 9 | "time" |
| 8 | 10 | |
| 9 | 11 | "github.com/mattn/go-sqlite3" |
| @@ -11,28 +13,338 @@ import ( | ||
| 11 | 13 | |
| 12 | 14 | const DSN = "_journal_mode=WAL&_busy_timeout=30000&_synchronous=NORMAL&_cache=shared" |
| 13 | 15 | |
| 14 | -func init() { | |
| 15 | - sql.Register("sqlite3_glean", &sqlite3.SQLiteDriver{ | |
| 16 | +type DB struct { | |
| 17 | + *sql.DB | |
| 18 | +} | |
| 19 | + | |
| 20 | +type Databases struct { | |
| 21 | + Users *UserStore | |
| 22 | + Articles *ArticleStore | |
| 23 | + | |
| 24 | + db *DB | |
| 25 | +} | |
| 26 | + | |
| 27 | +var multiDriverSeq int64 | |
| 28 | + | |
| 29 | +func OpenAll(basePath string) (*Databases, error) { | |
| 30 | + articlesPath := basePath + "_articles" | |
| 31 | + recsPath := basePath + "_recs" | |
| 32 | + | |
| 33 | + seq := atomic.AddInt64(&multiDriverSeq, 1) | |
| 34 | + driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq) | |
| 35 | + | |
| 36 | + sql.Register(driverName, &sqlite3.SQLiteDriver{ | |
| 16 | 37 | ConnectHook: func(conn *sqlite3.SQLiteConn) error { |
| 17 | - if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil { | |
| 38 | + if err := conn.RegisterFunc("exp", math.Exp, true); err != nil { | |
| 18 | 39 | return err |
| 19 | 40 | } |
| 20 | - if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil { | |
| 41 | + if err := conn.RegisterFunc("log", math.Log, true); err != nil { | |
| 21 | 42 | return err |
| 22 | 43 | } |
| 23 | - pragmas := []string{ | |
| 44 | + for _, p := range []string{ | |
| 24 | 45 | `PRAGMA wal_autocheckpoint = 1000`, |
| 25 | 46 | `PRAGMA temp_store = MEMORY`, |
| 26 | 47 | `PRAGMA mmap_size = 268435456`, |
| 27 | - } | |
| 28 | - for _, p := range pragmas { | |
| 48 | + } { | |
| 29 | 49 | if _, err := conn.Exec(p, nil); err != nil { |
| 30 | 50 | return err |
| 31 | 51 | } |
| 32 | 52 | } |
| 53 | + if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil { | |
| 54 | + return err | |
| 55 | + } | |
| 56 | + if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil { | |
| 57 | + return err | |
| 58 | + } | |
| 33 | 59 | return nil |
| 34 | 60 | }, |
| 35 | 61 | }) |
| 62 | + | |
| 63 | + db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN) | |
| 64 | + if err != nil { | |
| 65 | + return nil, err | |
| 66 | + } | |
| 67 | + db.SetMaxOpenConns(10) | |
| 68 | + db.SetMaxIdleConns(5) | |
| 69 | + db.SetConnMaxLifetime(30 * time.Minute) | |
| 70 | + d := &DB{db} | |
| 71 | + | |
| 72 | + if err := initUsersSchema(d); err != nil { | |
| 73 | + d.Close() | |
| 74 | + return nil, err | |
| 75 | + } | |
| 76 | + | |
| 77 | + if err := initArticlesSchema(d); err != nil { | |
| 78 | + d.Close() | |
| 79 | + return nil, err | |
| 80 | + } | |
| 81 | + | |
| 82 | + if err := initRecsSchema(d); err != nil { | |
| 83 | + d.Close() | |
| 84 | + return nil, err | |
| 85 | + } | |
| 86 | + | |
| 87 | + return &Databases{ | |
| 88 | + Users: NewUserStore(d), | |
| 89 | + Articles: NewArticleStore(d), | |
| 90 | + db: d, | |
| 91 | + }, nil | |
| 92 | +} | |
| 93 | + | |
| 94 | +func (d *Databases) Close() error { | |
| 95 | + if d.db != nil { | |
| 96 | + _ = d.db.Close() | |
| 97 | + } | |
| 98 | + return nil | |
| 99 | +} | |
| 100 | + | |
| 101 | +func (d *Databases) DB() *sql.DB { | |
| 102 | + return d.db.DB | |
| 103 | +} | |
| 104 | + | |
| 105 | +func initUsersSchema(db *DB) error { | |
| 106 | + for _, s := range usersSchema { | |
| 107 | + if _, err := db.Exec(s); err != nil { | |
| 108 | + return err | |
| 109 | + } | |
| 110 | + } | |
| 111 | + return nil | |
| 112 | +} | |
| 113 | + | |
| 114 | +func initArticlesSchema(db *DB) error { | |
| 115 | + for _, s := range articlesSchema { | |
| 116 | + if _, err := db.Exec(s); err != nil { | |
| 117 | + return err | |
| 118 | + } | |
| 119 | + } | |
| 120 | + return nil | |
| 121 | +} | |
| 122 | + | |
| 123 | +func initRecsSchema(db *DB) error { | |
| 124 | + for _, s := range recsSchema { | |
| 125 | + if _, err := db.Exec(s); err != nil { | |
| 126 | + return err | |
| 127 | + } | |
| 128 | + } | |
| 129 | + return nil | |
| 130 | +} | |
| 131 | + | |
| 132 | +var usersSchema = []string{ | |
| 133 | + `CREATE TABLE IF NOT EXISTS users ( | |
| 134 | + did TEXT PRIMARY KEY, | |
| 135 | + indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 136 | + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | |
| 137 | + )`, | |
| 138 | + | |
| 139 | + `CREATE TABLE IF NOT EXISTS follows ( | |
| 140 | + user_did TEXT NOT NULL, | |
| 141 | + target_did TEXT NOT NULL, | |
| 142 | + uri TEXT, | |
| 143 | + cid TEXT, | |
| 144 | + followed_at DATETIME, | |
| 145 | + PRIMARY KEY (user_did, target_did) | |
| 146 | + )`, | |
| 147 | + | |
| 148 | + `CREATE TABLE IF NOT EXISTS oauth_auth_requests ( | |
| 149 | + state TEXT PRIMARY KEY, | |
| 150 | + data TEXT NOT NULL | |
| 151 | + )`, | |
| 152 | + | |
| 153 | + `CREATE TABLE IF NOT EXISTS oauth_sessions ( | |
| 154 | + account_did TEXT NOT NULL, | |
| 155 | + session_id TEXT NOT NULL, | |
| 156 | + data TEXT NOT NULL, | |
| 157 | + PRIMARY KEY (account_did, session_id) | |
| 158 | + )`, | |
| 159 | + | |
| 160 | + `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`, | |
| 161 | + `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`, | |
| 162 | + `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`, | |
| 163 | + `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`, | |
| 164 | +} | |
| 165 | + | |
| 166 | +var articlesSchema = []string{ | |
| 167 | + `CREATE TABLE IF NOT EXISTS articles.feeds ( | |
| 168 | + feed_url TEXT PRIMARY KEY, | |
| 169 | + title TEXT, | |
| 170 | + site_url TEXT, | |
| 171 | + description TEXT, | |
| 172 | + feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')), | |
| 173 | + last_fetched_at DATETIME, | |
| 174 | + last_error TEXT, | |
| 175 | + subscriber_count INTEGER NOT NULL DEFAULT 0, | |
| 176 | + etag TEXT, | |
| 177 | + last_modified TEXT, | |
| 178 | + consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0, | |
| 179 | + error_count INTEGER NOT NULL DEFAULT 0, | |
| 180 | + favicon_url TEXT | |
| 181 | + )`, | |
| 182 | + | |
| 183 | + `CREATE TABLE IF NOT EXISTS articles.subscriptions ( | |
| 184 | + id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 185 | + user_did TEXT NOT NULL, | |
| 186 | + feed_url TEXT NOT NULL, | |
| 187 | + title TEXT, | |
| 188 | + category TEXT, | |
| 189 | + added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 190 | + uri TEXT, | |
| 191 | + cid TEXT, | |
| 192 | + UNIQUE(user_did, feed_url) | |
| 193 | + )`, | |
| 194 | + | |
| 195 | + `CREATE TABLE IF NOT EXISTS articles.articles ( | |
| 196 | + id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 197 | + feed_url TEXT NOT NULL, | |
| 198 | + guid TEXT NOT NULL, | |
| 199 | + title TEXT NOT NULL DEFAULT '', | |
| 200 | + url TEXT, | |
| 201 | + author TEXT, | |
| 202 | + summary TEXT, | |
| 203 | + content TEXT, | |
| 204 | + full_content TEXT, | |
| 205 | + published DATETIME, | |
| 206 | + updated DATETIME, | |
| 207 | + fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 208 | + UNIQUE(feed_url, guid) | |
| 209 | + )`, | |
| 210 | + | |
| 211 | + `CREATE TABLE IF NOT EXISTS articles.read_state ( | |
| 212 | + user_did TEXT NOT NULL, | |
| 213 | + article_id INTEGER NOT NULL, | |
| 214 | + is_read BOOLEAN NOT NULL DEFAULT 0, | |
| 215 | + read_at DATETIME, | |
| 216 | + PRIMARY KEY (user_did, article_id) | |
| 217 | + )`, | |
| 218 | + | |
| 219 | + `CREATE TABLE IF NOT EXISTS articles.annotations ( | |
| 220 | + id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 221 | + uri TEXT NOT NULL UNIQUE, | |
| 222 | + author_did TEXT NOT NULL, | |
| 223 | + feed_url TEXT NOT NULL, | |
| 224 | + article_url TEXT NOT NULL, | |
| 225 | + quote TEXT, | |
| 226 | + note TEXT, | |
| 227 | + tags TEXT, | |
| 228 | + rating INTEGER, | |
| 229 | + created_at DATETIME NOT NULL, | |
| 230 | + cid TEXT | |
| 231 | + )`, | |
| 232 | + | |
| 233 | + `CREATE TABLE IF NOT EXISTS articles.likes ( | |
| 234 | + id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 235 | + uri TEXT NOT NULL UNIQUE, | |
| 236 | + author_did TEXT NOT NULL, | |
| 237 | + feed_url TEXT NOT NULL, | |
| 238 | + article_url TEXT NOT NULL, | |
| 239 | + created_at DATETIME NOT NULL, | |
| 240 | + cid TEXT, | |
| 241 | + UNIQUE(author_did, feed_url, article_url) | |
| 242 | + )`, | |
| 243 | + | |
| 244 | + `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`, | |
| 245 | + `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`, | |
| 246 | + `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`, | |
| 247 | + `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`, | |
| 248 | + `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`, | |
| 249 | + `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`, | |
| 250 | + `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`, | |
| 251 | + `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`, | |
| 252 | + `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`, | |
| 253 | + `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`, | |
| 254 | + `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`, | |
| 255 | + `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`, | |
| 256 | + `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`, | |
| 257 | + `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`, | |
| 258 | + `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`, | |
| 259 | + | |
| 260 | + `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`, | |
| 261 | + `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN | |
| 262 | + INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author); | |
| 263 | + END`, | |
| 264 | + `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN | |
| 265 | + INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author); | |
| 266 | + END`, | |
| 267 | + `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN | |
| 268 | + INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author); | |
| 269 | + INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author); | |
| 270 | + END`, | |
| 271 | +} | |
| 272 | + | |
| 273 | +var recsSchema = []string{ | |
| 274 | + `CREATE TABLE IF NOT EXISTS recs.feed_similarity ( | |
| 275 | + feed_a TEXT NOT NULL, | |
| 276 | + feed_b TEXT NOT NULL, | |
| 277 | + jaccard REAL NOT NULL, | |
| 278 | + computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 279 | + PRIMARY KEY (feed_a, feed_b), | |
| 280 | + CHECK(feed_a < feed_b) | |
| 281 | + )`, | |
| 282 | + | |
| 283 | + `CREATE TABLE IF NOT EXISTS recs.user_similarity ( | |
| 284 | + user_a TEXT NOT NULL, | |
| 285 | + user_b TEXT NOT NULL, | |
| 286 | + jaccard REAL NOT NULL, | |
| 287 | + common_feeds INTEGER NOT NULL, | |
| 288 | + common_likes INTEGER NOT NULL DEFAULT 0, | |
| 289 | + common_tags INTEGER NOT NULL DEFAULT 0, | |
| 290 | + computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 291 | + PRIMARY KEY (user_a, user_b), | |
| 292 | + CHECK(user_a < user_b) | |
| 293 | + )`, | |
| 294 | + | |
| 295 | + `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations ( | |
| 296 | + user_did TEXT NOT NULL, | |
| 297 | + target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')), | |
| 298 | + target_id TEXT NOT NULL, | |
| 299 | + reason TEXT, | |
| 300 | + dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 301 | + PRIMARY KEY (user_did, target_type, target_id) | |
| 302 | + )`, | |
| 303 | + | |
| 304 | + `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions ( | |
| 305 | + user_did TEXT NOT NULL, | |
| 306 | + target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')), | |
| 307 | + target_id TEXT NOT NULL, | |
| 308 | + first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 309 | + last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 310 | + shown_count INTEGER NOT NULL DEFAULT 1, | |
| 311 | + acted BOOLEAN NOT NULL DEFAULT 0, | |
| 312 | + PRIMARY KEY (user_did, target_type, target_id) | |
| 313 | + )`, | |
| 314 | + | |
| 315 | + `CREATE TABLE IF NOT EXISTS recs.follow_distances ( | |
| 316 | + user_a TEXT NOT NULL, | |
| 317 | + user_b TEXT NOT NULL, | |
| 318 | + distance INTEGER NOT NULL CHECK(distance IN (1, 2)), | |
| 319 | + PRIMARY KEY (user_a, user_b) | |
| 320 | + )`, | |
| 321 | + | |
| 322 | + `CREATE TABLE IF NOT EXISTS recs.user_signal_weights ( | |
| 323 | + user_did TEXT PRIMARY KEY, | |
| 324 | + w_sub REAL NOT NULL DEFAULT 1.0, | |
| 325 | + w_like REAL NOT NULL DEFAULT 0.5, | |
| 326 | + w_tag REAL NOT NULL DEFAULT 0.3, | |
| 327 | + w_social REAL NOT NULL DEFAULT 0.7, | |
| 328 | + w_pop REAL NOT NULL DEFAULT 0.2, | |
| 329 | + w_category REAL NOT NULL DEFAULT 0.4, | |
| 330 | + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | |
| 331 | + )`, | |
| 332 | + | |
| 333 | + `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles ( | |
| 334 | + user_did TEXT PRIMARY KEY, | |
| 335 | + total_likes INTEGER NOT NULL DEFAULT 0, | |
| 336 | + total_tags INTEGER NOT NULL DEFAULT 0, | |
| 337 | + top_categories TEXT, | |
| 338 | + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | |
| 339 | + )`, | |
| 340 | + | |
| 341 | + `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`, | |
| 342 | + `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`, | |
| 343 | + `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`, | |
| 344 | + `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`, | |
| 345 | + `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`, | |
| 346 | + `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`, | |
| 347 | + `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`, | |
| 36 | 348 | } |
| 37 | 349 | |
| 38 | 350 | func NullStr(s string) sql.NullString { |
| @@ -53,7 +365,3 @@ func NullStrTags(tags []string) sql.NullString { | ||
| 53 | 365 | } |
| 54 | 366 | return sql.NullString{String: strings.Join(tags, ","), Valid: true} |
| 55 | 367 | } |
| 56 | - | |
| 57 | -type DB struct { | |
| 58 | - *sql.DB | |
| 59 | -} | |
| @@ -2,8 +2,10 @@ package db | |||
| 2 | 2 | ||
| 3 | import ( | 3 | import ( |
| 4 | "database/sql" | 4 | "database/sql" |
| 5 | + "fmt" | ||
| 5 | "math" | 6 | "math" |
| 6 | "strings" | 7 | "strings" |
| 8 | + "sync/atomic" | ||
| 7 | "time" | 9 | "time" |
| 8 | 10 | ||
| 9 | "github.com/mattn/go-sqlite3" | 11 | "github.com/mattn/go-sqlite3" |
| @@ -11,28 +13,338 @@ import ( | |||
| 11 | 13 | ||
| 12 | const DSN = "_journal_mode=WAL&_busy_timeout=30000&_synchronous=NORMAL&_cache=shared" | 14 | const DSN = "_journal_mode=WAL&_busy_timeout=30000&_synchronous=NORMAL&_cache=shared" |
| 13 | 15 | ||
| 14 | -func init() { | 16 | +type DB struct { |
| 15 | - sql.Register("sqlite3_glean", &sqlite3.SQLiteDriver{ | 17 | + *sql.DB |
| 18 | +} | ||
| 19 | + | ||
| 20 | +type Databases struct { | ||
| 21 | + Users *UserStore | ||
| 22 | + Articles *ArticleStore | ||
| 23 | + | ||
| 24 | + db *DB | ||
| 25 | +} | ||
| 26 | + | ||
| 27 | +var multiDriverSeq int64 | ||
| 28 | + | ||
| 29 | +func OpenAll(basePath string) (*Databases, error) { | ||
| 30 | + articlesPath := basePath + "_articles" | ||
| 31 | + recsPath := basePath + "_recs" | ||
| 32 | + | ||
| 33 | + seq := atomic.AddInt64(&multiDriverSeq, 1) | ||
| 34 | + driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq) | ||
| 35 | + | ||
| 36 | + sql.Register(driverName, &sqlite3.SQLiteDriver{ | ||
| 16 | ConnectHook: func(conn *sqlite3.SQLiteConn) error { | 37 | ConnectHook: func(conn *sqlite3.SQLiteConn) error { |
| 17 | - if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil { | 38 | + if err := conn.RegisterFunc("exp", math.Exp, true); err != nil { |
| 18 | return err | 39 | return err |
| 19 | } | 40 | } |
| 20 | - if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil { | 41 | + if err := conn.RegisterFunc("log", math.Log, true); err != nil { |
| 21 | return err | 42 | return err |
| 22 | } | 43 | } |
| 23 | - pragmas := []string{ | 44 | + for _, p := range []string{ |
| 24 | `PRAGMA wal_autocheckpoint = 1000`, | 45 | `PRAGMA wal_autocheckpoint = 1000`, |
| 25 | `PRAGMA temp_store = MEMORY`, | 46 | `PRAGMA temp_store = MEMORY`, |
| 26 | `PRAGMA mmap_size = 268435456`, | 47 | `PRAGMA mmap_size = 268435456`, |
| 27 | - } | 48 | + } { |
| 28 | - for _, p := range pragmas { | ||
| 29 | if _, err := conn.Exec(p, nil); err != nil { | 49 | if _, err := conn.Exec(p, nil); err != nil { |
| 30 | return err | 50 | return err |
| 31 | } | 51 | } |
| 32 | } | 52 | } |
| 53 | + if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil { | ||
| 54 | + return err | ||
| 55 | + } | ||
| 56 | + if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil { | ||
| 57 | + return err | ||
| 58 | + } | ||
| 33 | return nil | 59 | return nil |
| 34 | }, | 60 | }, |
| 35 | }) | 61 | }) |
| 62 | + | ||
| 63 | + db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN) | ||
| 64 | + if err != nil { | ||
| 65 | + return nil, err | ||
| 66 | + } | ||
| 67 | + db.SetMaxOpenConns(10) | ||
| 68 | + db.SetMaxIdleConns(5) | ||
| 69 | + db.SetConnMaxLifetime(30 * time.Minute) | ||
| 70 | + d := &DB{db} | ||
| 71 | + | ||
| 72 | + if err := initUsersSchema(d); err != nil { | ||
| 73 | + d.Close() | ||
| 74 | + return nil, err | ||
| 75 | + } | ||
| 76 | + | ||
| 77 | + if err := initArticlesSchema(d); err != nil { | ||
| 78 | + d.Close() | ||
| 79 | + return nil, err | ||
| 80 | + } | ||
| 81 | + | ||
| 82 | + if err := initRecsSchema(d); err != nil { | ||
| 83 | + d.Close() | ||
| 84 | + return nil, err | ||
| 85 | + } | ||
| 86 | + | ||
| 87 | + return &Databases{ | ||
| 88 | + Users: NewUserStore(d), | ||
| 89 | + Articles: NewArticleStore(d), | ||
| 90 | + db: d, | ||
| 91 | + }, nil | ||
| 92 | +} | ||
| 93 | + | ||
| 94 | +func (d *Databases) Close() error { | ||
| 95 | + if d.db != nil { | ||
| 96 | + _ = d.db.Close() | ||
| 97 | + } | ||
| 98 | + return nil | ||
| 99 | +} | ||
| 100 | + | ||
| 101 | +func (d *Databases) DB() *sql.DB { | ||
| 102 | + return d.db.DB | ||
| 103 | +} | ||
| 104 | + | ||
| 105 | +func initUsersSchema(db *DB) error { | ||
| 106 | + for _, s := range usersSchema { | ||
| 107 | + if _, err := db.Exec(s); err != nil { | ||
| 108 | + return err | ||
| 109 | + } | ||
| 110 | + } | ||
| 111 | + return nil | ||
| 112 | +} | ||
| 113 | + | ||
| 114 | +func initArticlesSchema(db *DB) error { | ||
| 115 | + for _, s := range articlesSchema { | ||
| 116 | + if _, err := db.Exec(s); err != nil { | ||
| 117 | + return err | ||
| 118 | + } | ||
| 119 | + } | ||
| 120 | + return nil | ||
| 121 | +} | ||
| 122 | + | ||
| 123 | +func initRecsSchema(db *DB) error { | ||
| 124 | + for _, s := range recsSchema { | ||
| 125 | + if _, err := db.Exec(s); err != nil { | ||
| 126 | + return err | ||
| 127 | + } | ||
| 128 | + } | ||
| 129 | + return nil | ||
| 130 | +} | ||
| 131 | + | ||
| 132 | +var usersSchema = []string{ | ||
| 133 | + `CREATE TABLE IF NOT EXISTS users ( | ||
| 134 | + did TEXT PRIMARY KEY, | ||
| 135 | + indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 136 | + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | ||
| 137 | + )`, | ||
| 138 | + | ||
| 139 | + `CREATE TABLE IF NOT EXISTS follows ( | ||
| 140 | + user_did TEXT NOT NULL, | ||
| 141 | + target_did TEXT NOT NULL, | ||
| 142 | + uri TEXT, | ||
| 143 | + cid TEXT, | ||
| 144 | + followed_at DATETIME, | ||
| 145 | + PRIMARY KEY (user_did, target_did) | ||
| 146 | + )`, | ||
| 147 | + | ||
| 148 | + `CREATE TABLE IF NOT EXISTS oauth_auth_requests ( | ||
| 149 | + state TEXT PRIMARY KEY, | ||
| 150 | + data TEXT NOT NULL | ||
| 151 | + )`, | ||
| 152 | + | ||
| 153 | + `CREATE TABLE IF NOT EXISTS oauth_sessions ( | ||
| 154 | + account_did TEXT NOT NULL, | ||
| 155 | + session_id TEXT NOT NULL, | ||
| 156 | + data TEXT NOT NULL, | ||
| 157 | + PRIMARY KEY (account_did, session_id) | ||
| 158 | + )`, | ||
| 159 | + | ||
| 160 | + `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`, | ||
| 161 | + `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`, | ||
| 162 | + `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`, | ||
| 163 | + `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`, | ||
| 164 | +} | ||
| 165 | + | ||
| 166 | +var articlesSchema = []string{ | ||
| 167 | + `CREATE TABLE IF NOT EXISTS articles.feeds ( | ||
| 168 | + feed_url TEXT PRIMARY KEY, | ||
| 169 | + title TEXT, | ||
| 170 | + site_url TEXT, | ||
| 171 | + description TEXT, | ||
| 172 | + feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')), | ||
| 173 | + last_fetched_at DATETIME, | ||
| 174 | + last_error TEXT, | ||
| 175 | + subscriber_count INTEGER NOT NULL DEFAULT 0, | ||
| 176 | + etag TEXT, | ||
| 177 | + last_modified TEXT, | ||
| 178 | + consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0, | ||
| 179 | + error_count INTEGER NOT NULL DEFAULT 0, | ||
| 180 | + favicon_url TEXT | ||
| 181 | + )`, | ||
| 182 | + | ||
| 183 | + `CREATE TABLE IF NOT EXISTS articles.subscriptions ( | ||
| 184 | + id INTEGER PRIMARY KEY AUTOINCREMENT, | ||
| 185 | + user_did TEXT NOT NULL, | ||
| 186 | + feed_url TEXT NOT NULL, | ||
| 187 | + title TEXT, | ||
| 188 | + category TEXT, | ||
| 189 | + added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 190 | + uri TEXT, | ||
| 191 | + cid TEXT, | ||
| 192 | + UNIQUE(user_did, feed_url) | ||
| 193 | + )`, | ||
| 194 | + | ||
| 195 | + `CREATE TABLE IF NOT EXISTS articles.articles ( | ||
| 196 | + id INTEGER PRIMARY KEY AUTOINCREMENT, | ||
| 197 | + feed_url TEXT NOT NULL, | ||
| 198 | + guid TEXT NOT NULL, | ||
| 199 | + title TEXT NOT NULL DEFAULT '', | ||
| 200 | + url TEXT, | ||
| 201 | + author TEXT, | ||
| 202 | + summary TEXT, | ||
| 203 | + content TEXT, | ||
| 204 | + full_content TEXT, | ||
| 205 | + published DATETIME, | ||
| 206 | + updated DATETIME, | ||
| 207 | + fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 208 | + UNIQUE(feed_url, guid) | ||
| 209 | + )`, | ||
| 210 | + | ||
| 211 | + `CREATE TABLE IF NOT EXISTS articles.read_state ( | ||
| 212 | + user_did TEXT NOT NULL, | ||
| 213 | + article_id INTEGER NOT NULL, | ||
| 214 | + is_read BOOLEAN NOT NULL DEFAULT 0, | ||
| 215 | + read_at DATETIME, | ||
| 216 | + PRIMARY KEY (user_did, article_id) | ||
| 217 | + )`, | ||
| 218 | + | ||
| 219 | + `CREATE TABLE IF NOT EXISTS articles.annotations ( | ||
| 220 | + id INTEGER PRIMARY KEY AUTOINCREMENT, | ||
| 221 | + uri TEXT NOT NULL UNIQUE, | ||
| 222 | + author_did TEXT NOT NULL, | ||
| 223 | + feed_url TEXT NOT NULL, | ||
| 224 | + article_url TEXT NOT NULL, | ||
| 225 | + quote TEXT, | ||
| 226 | + note TEXT, | ||
| 227 | + tags TEXT, | ||
| 228 | + rating INTEGER, | ||
| 229 | + created_at DATETIME NOT NULL, | ||
| 230 | + cid TEXT | ||
| 231 | + )`, | ||
| 232 | + | ||
| 233 | + `CREATE TABLE IF NOT EXISTS articles.likes ( | ||
| 234 | + id INTEGER PRIMARY KEY AUTOINCREMENT, | ||
| 235 | + uri TEXT NOT NULL UNIQUE, | ||
| 236 | + author_did TEXT NOT NULL, | ||
| 237 | + feed_url TEXT NOT NULL, | ||
| 238 | + article_url TEXT NOT NULL, | ||
| 239 | + created_at DATETIME NOT NULL, | ||
| 240 | + cid TEXT, | ||
| 241 | + UNIQUE(author_did, feed_url, article_url) | ||
| 242 | + )`, | ||
| 243 | + | ||
| 244 | + `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`, | ||
| 245 | + `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`, | ||
| 246 | + `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`, | ||
| 247 | + `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`, | ||
| 248 | + `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`, | ||
| 249 | + `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`, | ||
| 250 | + `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`, | ||
| 251 | + `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`, | ||
| 252 | + `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`, | ||
| 253 | + `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`, | ||
| 254 | + `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`, | ||
| 255 | + `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`, | ||
| 256 | + `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`, | ||
| 257 | + `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`, | ||
| 258 | + `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`, | ||
| 259 | + | ||
| 260 | + `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`, | ||
| 261 | + `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN | ||
| 262 | + INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author); | ||
| 263 | + END`, | ||
| 264 | + `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN | ||
| 265 | + INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author); | ||
| 266 | + END`, | ||
| 267 | + `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN | ||
| 268 | + INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author); | ||
| 269 | + INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author); | ||
| 270 | + END`, | ||
| 271 | +} | ||
| 272 | + | ||
| 273 | +var recsSchema = []string{ | ||
| 274 | + `CREATE TABLE IF NOT EXISTS recs.feed_similarity ( | ||
| 275 | + feed_a TEXT NOT NULL, | ||
| 276 | + feed_b TEXT NOT NULL, | ||
| 277 | + jaccard REAL NOT NULL, | ||
| 278 | + computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 279 | + PRIMARY KEY (feed_a, feed_b), | ||
| 280 | + CHECK(feed_a < feed_b) | ||
| 281 | + )`, | ||
| 282 | + | ||
| 283 | + `CREATE TABLE IF NOT EXISTS recs.user_similarity ( | ||
| 284 | + user_a TEXT NOT NULL, | ||
| 285 | + user_b TEXT NOT NULL, | ||
| 286 | + jaccard REAL NOT NULL, | ||
| 287 | + common_feeds INTEGER NOT NULL, | ||
| 288 | + common_likes INTEGER NOT NULL DEFAULT 0, | ||
| 289 | + common_tags INTEGER NOT NULL DEFAULT 0, | ||
| 290 | + computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 291 | + PRIMARY KEY (user_a, user_b), | ||
| 292 | + CHECK(user_a < user_b) | ||
| 293 | + )`, | ||
| 294 | + | ||
| 295 | + `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations ( | ||
| 296 | + user_did TEXT NOT NULL, | ||
| 297 | + target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')), | ||
| 298 | + target_id TEXT NOT NULL, | ||
| 299 | + reason TEXT, | ||
| 300 | + dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 301 | + PRIMARY KEY (user_did, target_type, target_id) | ||
| 302 | + )`, | ||
| 303 | + | ||
| 304 | + `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions ( | ||
| 305 | + user_did TEXT NOT NULL, | ||
| 306 | + target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')), | ||
| 307 | + target_id TEXT NOT NULL, | ||
| 308 | + first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 309 | + last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 310 | + shown_count INTEGER NOT NULL DEFAULT 1, | ||
| 311 | + acted BOOLEAN NOT NULL DEFAULT 0, | ||
| 312 | + PRIMARY KEY (user_did, target_type, target_id) | ||
| 313 | + )`, | ||
| 314 | + | ||
| 315 | + `CREATE TABLE IF NOT EXISTS recs.follow_distances ( | ||
| 316 | + user_a TEXT NOT NULL, | ||
| 317 | + user_b TEXT NOT NULL, | ||
| 318 | + distance INTEGER NOT NULL CHECK(distance IN (1, 2)), | ||
| 319 | + PRIMARY KEY (user_a, user_b) | ||
| 320 | + )`, | ||
| 321 | + | ||
| 322 | + `CREATE TABLE IF NOT EXISTS recs.user_signal_weights ( | ||
| 323 | + user_did TEXT PRIMARY KEY, | ||
| 324 | + w_sub REAL NOT NULL DEFAULT 1.0, | ||
| 325 | + w_like REAL NOT NULL DEFAULT 0.5, | ||
| 326 | + w_tag REAL NOT NULL DEFAULT 0.3, | ||
| 327 | + w_social REAL NOT NULL DEFAULT 0.7, | ||
| 328 | + w_pop REAL NOT NULL DEFAULT 0.2, | ||
| 329 | + w_category REAL NOT NULL DEFAULT 0.4, | ||
| 330 | + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | ||
| 331 | + )`, | ||
| 332 | + | ||
| 333 | + `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles ( | ||
| 334 | + user_did TEXT PRIMARY KEY, | ||
| 335 | + total_likes INTEGER NOT NULL DEFAULT 0, | ||
| 336 | + total_tags INTEGER NOT NULL DEFAULT 0, | ||
| 337 | + top_categories TEXT, | ||
| 338 | + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | ||
| 339 | + )`, | ||
| 340 | + | ||
| 341 | + `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`, | ||
| 342 | + `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`, | ||
| 343 | + `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`, | ||
| 344 | + `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`, | ||
| 345 | + `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`, | ||
| 346 | + `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`, | ||
| 347 | + `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`, | ||
| 36 | } | 348 | } |
| 37 | 349 | ||
| 38 | func NullStr(s string) sql.NullString { | 350 | func NullStr(s string) sql.NullString { |
| @@ -53,7 +365,3 @@ func NullStrTags(tags []string) sql.NullString { | |||
| 53 | } | 365 | } |
| 54 | return sql.NullString{String: strings.Join(tags, ","), Valid: true} | 366 | return sql.NullString{String: strings.Join(tags, ","), Valid: true} |
| 55 | } | 367 | } |
| 56 | - | ||
| 57 | -type DB struct { | ||
| 58 | - *sql.DB | ||
| 59 | -} | ||
modified
internal/db/feed.go +5 -8 | @@ -54,9 +54,6 @@ type Subscription struct { | ||
| 54 | 54 | FaviconURL sql.NullString |
| 55 | 55 | } |
| 56 | 56 | |
| 57 | -const feedSelectCols = `feed_url, title, site_url, description, feed_type, | |
| 58 | - last_fetched_at, last_error, subscriber_count, etag, last_modified, | |
| 59 | - consecutive_empty_fetches, error_count, favicon_url` | |
| 60 | 57 | |
| 61 | 58 | func scanFeed(scanner interface{ Scan(...any) error }) (*Feed, error) { |
| 62 | 59 | f := &Feed{} |
| @@ -73,13 +70,13 @@ func (s *ArticleStore) UpsertFeed(ctx context.Context, feed *Feed) error { | ||
| 73 | 70 | } |
| 74 | 71 | |
| 75 | 72 | func (s *ArticleStore) GetFeed(ctx context.Context, feedURL string) (*Feed, error) { |
| 76 | - row := s.db.QueryRowContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds WHERE feed_url = ?`, feedURL) | |
| 73 | + row := s.db.QueryRowContext(ctx, `SELECT * FROM articles.feeds WHERE feed_url = ?`, feedURL) | |
| 77 | 74 | return scanFeed(row) |
| 78 | 75 | } |
| 79 | 76 | |
| 80 | 77 | func (s *ArticleStore) GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error) { |
| 81 | 78 | cutoff := time.Now().Add(-olderThan) |
| 82 | - rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds | |
| 79 | + rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds | |
| 83 | 80 | WHERE subscriber_count > 0 AND error_count < 25 AND (last_fetched_at IS NULL OR last_fetched_at <= ?) |
| 84 | 81 | ORDER BY last_fetched_at ASC NULLS FIRST LIMIT ?`, cutoff, limit) |
| 85 | 82 | if err != nil { |
| @@ -309,7 +306,7 @@ func (s *ArticleStore) UpdateFeedFavicon(ctx context.Context, feedURL, faviconUR | ||
| 309 | 306 | } |
| 310 | 307 | |
| 311 | 308 | func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, threshold int) ([]*Feed, error) { |
| 312 | - rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds f | |
| 309 | + rows, err := s.db.QueryContext(ctx, `SELECT f.* FROM articles.feeds f | |
| 313 | 310 | JOIN articles.subscriptions s ON s.feed_url = f.feed_url AND s.user_did = ? |
| 314 | 311 | WHERE f.error_count >= ? ORDER BY f.error_count DESC`, userDID, threshold) |
| 315 | 312 | if err != nil { |
| @@ -329,7 +326,7 @@ func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, thresh | ||
| 329 | 326 | } |
| 330 | 327 | |
| 331 | 328 | func (s *ArticleStore) ListAllFeeds(ctx context.Context, limit, offset int) ([]*Feed, error) { |
| 332 | - rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds | |
| 329 | + rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds | |
| 333 | 330 | ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, limit, offset) |
| 334 | 331 | if err != nil { |
| 335 | 332 | return nil, err |
| @@ -459,7 +456,7 @@ func (s *ArticleStore) BatchReconcileSubscriptions(ctx context.Context, userDID | ||
| 459 | 456 | } |
| 460 | 457 | |
| 461 | 458 | func (s *ArticleStore) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) { |
| 462 | - rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds | |
| 459 | + rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds | |
| 463 | 460 | WHERE feed_url NOT IN (SELECT feed_url FROM articles.subscriptions WHERE user_did = ?) |
| 464 | 461 | ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, userDID, limit, offset) |
| 465 | 462 | if err != nil { |
| @@ -54,9 +54,6 @@ type Subscription struct { | |||
| 54 | FaviconURL sql.NullString | 54 | FaviconURL sql.NullString |
| 55 | } | 55 | } |
| 56 | 56 | ||
| 57 | -const feedSelectCols = `feed_url, title, site_url, description, feed_type, | ||
| 58 | - last_fetched_at, last_error, subscriber_count, etag, last_modified, | ||
| 59 | - consecutive_empty_fetches, error_count, favicon_url` | ||
| 60 | 57 | ||
| 61 | func scanFeed(scanner interface{ Scan(...any) error }) (*Feed, error) { | 58 | func scanFeed(scanner interface{ Scan(...any) error }) (*Feed, error) { |
| 62 | f := &Feed{} | 59 | f := &Feed{} |
| @@ -73,13 +70,13 @@ func (s *ArticleStore) UpsertFeed(ctx context.Context, feed *Feed) error { | |||
| 73 | } | 70 | } |
| 74 | 71 | ||
| 75 | func (s *ArticleStore) GetFeed(ctx context.Context, feedURL string) (*Feed, error) { | 72 | func (s *ArticleStore) GetFeed(ctx context.Context, feedURL string) (*Feed, error) { |
| 76 | - row := s.db.QueryRowContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds WHERE feed_url = ?`, feedURL) | 73 | + row := s.db.QueryRowContext(ctx, `SELECT * FROM articles.feeds WHERE feed_url = ?`, feedURL) |
| 77 | return scanFeed(row) | 74 | return scanFeed(row) |
| 78 | } | 75 | } |
| 79 | 76 | ||
| 80 | func (s *ArticleStore) GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error) { | 77 | func (s *ArticleStore) GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error) { |
| 81 | cutoff := time.Now().Add(-olderThan) | 78 | cutoff := time.Now().Add(-olderThan) |
| 82 | - rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds | 79 | + rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds |
| 83 | WHERE subscriber_count > 0 AND error_count < 25 AND (last_fetched_at IS NULL OR last_fetched_at <= ?) | 80 | WHERE subscriber_count > 0 AND error_count < 25 AND (last_fetched_at IS NULL OR last_fetched_at <= ?) |
| 84 | ORDER BY last_fetched_at ASC NULLS FIRST LIMIT ?`, cutoff, limit) | 81 | ORDER BY last_fetched_at ASC NULLS FIRST LIMIT ?`, cutoff, limit) |
| 85 | if err != nil { | 82 | if err != nil { |
| @@ -309,7 +306,7 @@ func (s *ArticleStore) UpdateFeedFavicon(ctx context.Context, feedURL, faviconUR | |||
| 309 | } | 306 | } |
| 310 | 307 | ||
| 311 | func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, threshold int) ([]*Feed, error) { | 308 | func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, threshold int) ([]*Feed, error) { |
| 312 | - rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds f | 309 | + rows, err := s.db.QueryContext(ctx, `SELECT f.* FROM articles.feeds f |
| 313 | JOIN articles.subscriptions s ON s.feed_url = f.feed_url AND s.user_did = ? | 310 | JOIN articles.subscriptions s ON s.feed_url = f.feed_url AND s.user_did = ? |
| 314 | WHERE f.error_count >= ? ORDER BY f.error_count DESC`, userDID, threshold) | 311 | WHERE f.error_count >= ? ORDER BY f.error_count DESC`, userDID, threshold) |
| 315 | if err != nil { | 312 | if err != nil { |
| @@ -329,7 +326,7 @@ func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, thresh | |||
| 329 | } | 326 | } |
| 330 | 327 | ||
| 331 | func (s *ArticleStore) ListAllFeeds(ctx context.Context, limit, offset int) ([]*Feed, error) { | 328 | func (s *ArticleStore) ListAllFeeds(ctx context.Context, limit, offset int) ([]*Feed, error) { |
| 332 | - rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds | 329 | + rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds |
| 333 | ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, limit, offset) | 330 | ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, limit, offset) |
| 334 | if err != nil { | 331 | if err != nil { |
| 335 | return nil, err | 332 | return nil, err |
| @@ -459,7 +456,7 @@ func (s *ArticleStore) BatchReconcileSubscriptions(ctx context.Context, userDID | |||
| 459 | } | 456 | } |
| 460 | 457 | ||
| 461 | func (s *ArticleStore) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) { | 458 | func (s *ArticleStore) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) { |
| 462 | - rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds | 459 | + rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds |
| 463 | WHERE feed_url NOT IN (SELECT feed_url FROM articles.subscriptions WHERE user_did = ?) | 460 | WHERE feed_url NOT IN (SELECT feed_url FROM articles.subscriptions WHERE user_did = ?) |
| 464 | ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, userDID, limit, offset) | 461 | ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, userDID, limit, offset) |
| 465 | if err != nil { | 462 | if err != nil { |
modified
internal/db/follow.go +1 -2 | @@ -60,8 +60,7 @@ func (s *UserStore) ListFollows(ctx context.Context, userDID string, limit, offs | ||
| 60 | 60 | |
| 61 | 61 | func (s *UserStore) ListFollowers(ctx context.Context, targetDID string, limit, offset int) ([]*Follow, error) { |
| 62 | 62 | rows, err := s.db.QueryContext(ctx, ` |
| 63 | - SELECT user_did, target_did, uri, cid, followed_at | |
| 64 | - FROM follows WHERE target_did = ? | |
| 63 | + SELECT * FROM follows WHERE target_did = ? | |
| 65 | 64 | ORDER BY followed_at DESC |
| 66 | 65 | LIMIT ? OFFSET ? |
| 67 | 66 | `, targetDID, limit, offset) |
| @@ -60,8 +60,7 @@ func (s *UserStore) ListFollows(ctx context.Context, userDID string, limit, offs | |||
| 60 | 60 | ||
| 61 | func (s *UserStore) ListFollowers(ctx context.Context, targetDID string, limit, offset int) ([]*Follow, error) { | 61 | func (s *UserStore) ListFollowers(ctx context.Context, targetDID string, limit, offset int) ([]*Follow, error) { |
| 62 | rows, err := s.db.QueryContext(ctx, ` | 62 | rows, err := s.db.QueryContext(ctx, ` |
| 63 | - SELECT user_did, target_did, uri, cid, followed_at | 63 | + SELECT * FROM follows WHERE target_did = ? |
| 64 | - FROM follows WHERE target_did = ? | ||
| 65 | ORDER BY followed_at DESC | 64 | ORDER BY followed_at DESC |
| 66 | LIMIT ? OFFSET ? | 65 | LIMIT ? OFFSET ? |
| 67 | `, targetDID, limit, offset) | 66 | `, targetDID, limit, offset) |
deleted
internal/db/multi.go +0 -341 | deleted file mode 100644 | ||
| @@ -1,341 +0,0 @@ | ||
| 1 | -package db | |
| 2 | - | |
| 3 | -import ( | |
| 4 | - "database/sql" | |
| 5 | - "fmt" | |
| 6 | - "math" | |
| 7 | - "sync/atomic" | |
| 8 | - "time" | |
| 9 | - | |
| 10 | - "github.com/mattn/go-sqlite3" | |
| 11 | -) | |
| 12 | - | |
| 13 | -type Databases struct { | |
| 14 | - Users *UserStore | |
| 15 | - Articles *ArticleStore | |
| 16 | - | |
| 17 | - db *DB | |
| 18 | -} | |
| 19 | - | |
| 20 | -var multiDriverSeq int64 | |
| 21 | - | |
| 22 | -func OpenAll(basePath string) (*Databases, error) { | |
| 23 | - articlesPath := basePath + "_articles" | |
| 24 | - recsPath := basePath + "_recs" | |
| 25 | - | |
| 26 | - seq := atomic.AddInt64(&multiDriverSeq, 1) | |
| 27 | - driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq) | |
| 28 | - | |
| 29 | - sql.Register(driverName, &sqlite3.SQLiteDriver{ | |
| 30 | - ConnectHook: func(conn *sqlite3.SQLiteConn) error { | |
| 31 | - if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil { | |
| 32 | - return err | |
| 33 | - } | |
| 34 | - if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil { | |
| 35 | - return err | |
| 36 | - } | |
| 37 | - for _, p := range []string{ | |
| 38 | - `PRAGMA wal_autocheckpoint = 1000`, | |
| 39 | - `PRAGMA temp_store = MEMORY`, | |
| 40 | - `PRAGMA mmap_size = 268435456`, | |
| 41 | - } { | |
| 42 | - if _, err := conn.Exec(p, nil); err != nil { | |
| 43 | - return err | |
| 44 | - } | |
| 45 | - } | |
| 46 | - if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil { | |
| 47 | - return err | |
| 48 | - } | |
| 49 | - if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil { | |
| 50 | - return err | |
| 51 | - } | |
| 52 | - return nil | |
| 53 | - }, | |
| 54 | - }) | |
| 55 | - | |
| 56 | - db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN) | |
| 57 | - if err != nil { | |
| 58 | - return nil, err | |
| 59 | - } | |
| 60 | - db.SetMaxOpenConns(10) | |
| 61 | - db.SetMaxIdleConns(5) | |
| 62 | - db.SetConnMaxLifetime(30 * time.Minute) | |
| 63 | - d := &DB{db} | |
| 64 | - | |
| 65 | - if err := initUsersSchema(d); err != nil { | |
| 66 | - d.Close() | |
| 67 | - return nil, err | |
| 68 | - } | |
| 69 | - | |
| 70 | - if err := initArticlesSchema(d); err != nil { | |
| 71 | - d.Close() | |
| 72 | - return nil, err | |
| 73 | - } | |
| 74 | - | |
| 75 | - if err := initRecsSchema(d); err != nil { | |
| 76 | - d.Close() | |
| 77 | - return nil, err | |
| 78 | - } | |
| 79 | - | |
| 80 | - return &Databases{ | |
| 81 | - Users: NewUserStore(d), | |
| 82 | - Articles: NewArticleStore(d), | |
| 83 | - db: d, | |
| 84 | - }, nil | |
| 85 | -} | |
| 86 | - | |
| 87 | -func (d *Databases) Close() error { | |
| 88 | - if d.db != nil { | |
| 89 | - _ = d.db.Close() | |
| 90 | - } | |
| 91 | - return nil | |
| 92 | -} | |
| 93 | - | |
| 94 | -func (d *Databases) DB() *sql.DB { | |
| 95 | - return d.db.DB | |
| 96 | -} | |
| 97 | - | |
| 98 | -func initUsersSchema(db *DB) error { | |
| 99 | - for _, s := range usersSchema { | |
| 100 | - if _, err := db.Exec(s); err != nil { | |
| 101 | - return err | |
| 102 | - } | |
| 103 | - } | |
| 104 | - return nil | |
| 105 | -} | |
| 106 | - | |
| 107 | -func initArticlesSchema(db *DB) error { | |
| 108 | - for _, s := range articlesSchema { | |
| 109 | - if _, err := db.Exec(s); err != nil { | |
| 110 | - return err | |
| 111 | - } | |
| 112 | - } | |
| 113 | - return nil | |
| 114 | -} | |
| 115 | - | |
| 116 | -func initRecsSchema(db *DB) error { | |
| 117 | - for _, s := range recsSchema { | |
| 118 | - if _, err := db.Exec(s); err != nil { | |
| 119 | - return err | |
| 120 | - } | |
| 121 | - } | |
| 122 | - return nil | |
| 123 | -} | |
| 124 | - | |
| 125 | -var usersSchema = []string{ | |
| 126 | - `CREATE TABLE IF NOT EXISTS users ( | |
| 127 | - did TEXT PRIMARY KEY, | |
| 128 | - indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 129 | - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | |
| 130 | - )`, | |
| 131 | - | |
| 132 | - `CREATE TABLE IF NOT EXISTS follows ( | |
| 133 | - user_did TEXT NOT NULL, | |
| 134 | - target_did TEXT NOT NULL, | |
| 135 | - uri TEXT, | |
| 136 | - cid TEXT, | |
| 137 | - followed_at DATETIME, | |
| 138 | - PRIMARY KEY (user_did, target_did) | |
| 139 | - )`, | |
| 140 | - | |
| 141 | - `CREATE TABLE IF NOT EXISTS oauth_auth_requests ( | |
| 142 | - state TEXT PRIMARY KEY, | |
| 143 | - data TEXT NOT NULL | |
| 144 | - )`, | |
| 145 | - | |
| 146 | - `CREATE TABLE IF NOT EXISTS oauth_sessions ( | |
| 147 | - account_did TEXT NOT NULL, | |
| 148 | - session_id TEXT NOT NULL, | |
| 149 | - data TEXT NOT NULL, | |
| 150 | - PRIMARY KEY (account_did, session_id) | |
| 151 | - )`, | |
| 152 | - | |
| 153 | - `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`, | |
| 154 | - `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`, | |
| 155 | - `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`, | |
| 156 | - `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`, | |
| 157 | -} | |
| 158 | - | |
| 159 | -var articlesSchema = []string{ | |
| 160 | - `CREATE TABLE IF NOT EXISTS articles.feeds ( | |
| 161 | - feed_url TEXT PRIMARY KEY, | |
| 162 | - title TEXT, | |
| 163 | - site_url TEXT, | |
| 164 | - description TEXT, | |
| 165 | - feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')), | |
| 166 | - last_fetched_at DATETIME, | |
| 167 | - last_error TEXT, | |
| 168 | - subscriber_count INTEGER NOT NULL DEFAULT 0, | |
| 169 | - etag TEXT, | |
| 170 | - last_modified TEXT, | |
| 171 | - consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0, | |
| 172 | - error_count INTEGER NOT NULL DEFAULT 0, | |
| 173 | - favicon_url TEXT | |
| 174 | - )`, | |
| 175 | - | |
| 176 | - `CREATE TABLE IF NOT EXISTS articles.subscriptions ( | |
| 177 | - id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 178 | - user_did TEXT NOT NULL, | |
| 179 | - feed_url TEXT NOT NULL, | |
| 180 | - title TEXT, | |
| 181 | - category TEXT, | |
| 182 | - added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 183 | - uri TEXT, | |
| 184 | - cid TEXT, | |
| 185 | - UNIQUE(user_did, feed_url) | |
| 186 | - )`, | |
| 187 | - | |
| 188 | - `CREATE TABLE IF NOT EXISTS articles.articles ( | |
| 189 | - id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 190 | - feed_url TEXT NOT NULL, | |
| 191 | - guid TEXT NOT NULL, | |
| 192 | - title TEXT NOT NULL DEFAULT '', | |
| 193 | - url TEXT, | |
| 194 | - author TEXT, | |
| 195 | - summary TEXT, | |
| 196 | - content TEXT, | |
| 197 | - full_content TEXT, | |
| 198 | - published DATETIME, | |
| 199 | - updated DATETIME, | |
| 200 | - fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 201 | - UNIQUE(feed_url, guid) | |
| 202 | - )`, | |
| 203 | - | |
| 204 | - `CREATE TABLE IF NOT EXISTS articles.read_state ( | |
| 205 | - user_did TEXT NOT NULL, | |
| 206 | - article_id INTEGER NOT NULL, | |
| 207 | - is_read BOOLEAN NOT NULL DEFAULT 0, | |
| 208 | - read_at DATETIME, | |
| 209 | - PRIMARY KEY (user_did, article_id) | |
| 210 | - )`, | |
| 211 | - | |
| 212 | - `CREATE TABLE IF NOT EXISTS articles.annotations ( | |
| 213 | - id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 214 | - uri TEXT NOT NULL UNIQUE, | |
| 215 | - author_did TEXT NOT NULL, | |
| 216 | - feed_url TEXT NOT NULL, | |
| 217 | - article_url TEXT NOT NULL, | |
| 218 | - quote TEXT, | |
| 219 | - note TEXT, | |
| 220 | - tags TEXT, | |
| 221 | - rating INTEGER, | |
| 222 | - created_at DATETIME NOT NULL, | |
| 223 | - cid TEXT | |
| 224 | - )`, | |
| 225 | - | |
| 226 | - `CREATE TABLE IF NOT EXISTS articles.likes ( | |
| 227 | - id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 228 | - uri TEXT NOT NULL UNIQUE, | |
| 229 | - author_did TEXT NOT NULL, | |
| 230 | - feed_url TEXT NOT NULL, | |
| 231 | - article_url TEXT NOT NULL, | |
| 232 | - created_at DATETIME NOT NULL, | |
| 233 | - cid TEXT, | |
| 234 | - UNIQUE(author_did, feed_url, article_url) | |
| 235 | - )`, | |
| 236 | - | |
| 237 | - `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`, | |
| 238 | - `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`, | |
| 239 | - `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`, | |
| 240 | - `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`, | |
| 241 | - `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`, | |
| 242 | - `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`, | |
| 243 | - `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`, | |
| 244 | - `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`, | |
| 245 | - `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`, | |
| 246 | - `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`, | |
| 247 | - `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`, | |
| 248 | - `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`, | |
| 249 | - `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`, | |
| 250 | - `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`, | |
| 251 | - `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`, | |
| 252 | - | |
| 253 | - `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`, | |
| 254 | - `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN | |
| 255 | - INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author); | |
| 256 | - END`, | |
| 257 | - `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN | |
| 258 | - INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author); | |
| 259 | - END`, | |
| 260 | - `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN | |
| 261 | - INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author); | |
| 262 | - INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author); | |
| 263 | - END`, | |
| 264 | -} | |
| 265 | - | |
| 266 | -var recsSchema = []string{ | |
| 267 | - `CREATE TABLE IF NOT EXISTS recs.feed_similarity ( | |
| 268 | - feed_a TEXT NOT NULL, | |
| 269 | - feed_b TEXT NOT NULL, | |
| 270 | - jaccard REAL NOT NULL, | |
| 271 | - computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 272 | - PRIMARY KEY (feed_a, feed_b), | |
| 273 | - CHECK(feed_a < feed_b) | |
| 274 | - )`, | |
| 275 | - | |
| 276 | - `CREATE TABLE IF NOT EXISTS recs.user_similarity ( | |
| 277 | - user_a TEXT NOT NULL, | |
| 278 | - user_b TEXT NOT NULL, | |
| 279 | - jaccard REAL NOT NULL, | |
| 280 | - common_feeds INTEGER NOT NULL, | |
| 281 | - common_likes INTEGER NOT NULL DEFAULT 0, | |
| 282 | - common_tags INTEGER NOT NULL DEFAULT 0, | |
| 283 | - computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 284 | - PRIMARY KEY (user_a, user_b), | |
| 285 | - CHECK(user_a < user_b) | |
| 286 | - )`, | |
| 287 | - | |
| 288 | - `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations ( | |
| 289 | - user_did TEXT NOT NULL, | |
| 290 | - target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')), | |
| 291 | - target_id TEXT NOT NULL, | |
| 292 | - reason TEXT, | |
| 293 | - dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 294 | - PRIMARY KEY (user_did, target_type, target_id) | |
| 295 | - )`, | |
| 296 | - | |
| 297 | - `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions ( | |
| 298 | - user_did TEXT NOT NULL, | |
| 299 | - target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')), | |
| 300 | - target_id TEXT NOT NULL, | |
| 301 | - first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 302 | - last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | |
| 303 | - shown_count INTEGER NOT NULL DEFAULT 1, | |
| 304 | - acted BOOLEAN NOT NULL DEFAULT 0, | |
| 305 | - PRIMARY KEY (user_did, target_type, target_id) | |
| 306 | - )`, | |
| 307 | - | |
| 308 | - `CREATE TABLE IF NOT EXISTS recs.follow_distances ( | |
| 309 | - user_a TEXT NOT NULL, | |
| 310 | - user_b TEXT NOT NULL, | |
| 311 | - distance INTEGER NOT NULL CHECK(distance IN (1, 2)), | |
| 312 | - PRIMARY KEY (user_a, user_b) | |
| 313 | - )`, | |
| 314 | - | |
| 315 | - `CREATE TABLE IF NOT EXISTS recs.user_signal_weights ( | |
| 316 | - user_did TEXT PRIMARY KEY, | |
| 317 | - w_sub REAL NOT NULL DEFAULT 1.0, | |
| 318 | - w_like REAL NOT NULL DEFAULT 0.5, | |
| 319 | - w_tag REAL NOT NULL DEFAULT 0.3, | |
| 320 | - w_social REAL NOT NULL DEFAULT 0.7, | |
| 321 | - w_pop REAL NOT NULL DEFAULT 0.2, | |
| 322 | - w_category REAL NOT NULL DEFAULT 0.4, | |
| 323 | - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | |
| 324 | - )`, | |
| 325 | - | |
| 326 | - `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles ( | |
| 327 | - user_did TEXT PRIMARY KEY, | |
| 328 | - total_likes INTEGER NOT NULL DEFAULT 0, | |
| 329 | - total_tags INTEGER NOT NULL DEFAULT 0, | |
| 330 | - top_categories TEXT, | |
| 331 | - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | |
| 332 | - )`, | |
| 333 | - | |
| 334 | - `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`, | |
| 335 | - `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`, | |
| 336 | - `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`, | |
| 337 | - `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`, | |
| 338 | - `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`, | |
| 339 | - `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`, | |
| 340 | - `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`, | |
| 341 | -} | |
| deleted file mode 100644 | |||
| @@ -1,341 +0,0 @@ | |||
| 1 | -package db | ||
| 2 | - | ||
| 3 | -import ( | ||
| 4 | - "database/sql" | ||
| 5 | - "fmt" | ||
| 6 | - "math" | ||
| 7 | - "sync/atomic" | ||
| 8 | - "time" | ||
| 9 | - | ||
| 10 | - "github.com/mattn/go-sqlite3" | ||
| 11 | -) | ||
| 12 | - | ||
| 13 | -type Databases struct { | ||
| 14 | - Users *UserStore | ||
| 15 | - Articles *ArticleStore | ||
| 16 | - | ||
| 17 | - db *DB | ||
| 18 | -} | ||
| 19 | - | ||
| 20 | -var multiDriverSeq int64 | ||
| 21 | - | ||
| 22 | -func OpenAll(basePath string) (*Databases, error) { | ||
| 23 | - articlesPath := basePath + "_articles" | ||
| 24 | - recsPath := basePath + "_recs" | ||
| 25 | - | ||
| 26 | - seq := atomic.AddInt64(&multiDriverSeq, 1) | ||
| 27 | - driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq) | ||
| 28 | - | ||
| 29 | - sql.Register(driverName, &sqlite3.SQLiteDriver{ | ||
| 30 | - ConnectHook: func(conn *sqlite3.SQLiteConn) error { | ||
| 31 | - if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil { | ||
| 32 | - return err | ||
| 33 | - } | ||
| 34 | - if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil { | ||
| 35 | - return err | ||
| 36 | - } | ||
| 37 | - for _, p := range []string{ | ||
| 38 | - `PRAGMA wal_autocheckpoint = 1000`, | ||
| 39 | - `PRAGMA temp_store = MEMORY`, | ||
| 40 | - `PRAGMA mmap_size = 268435456`, | ||
| 41 | - } { | ||
| 42 | - if _, err := conn.Exec(p, nil); err != nil { | ||
| 43 | - return err | ||
| 44 | - } | ||
| 45 | - } | ||
| 46 | - if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil { | ||
| 47 | - return err | ||
| 48 | - } | ||
| 49 | - if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil { | ||
| 50 | - return err | ||
| 51 | - } | ||
| 52 | - return nil | ||
| 53 | - }, | ||
| 54 | - }) | ||
| 55 | - | ||
| 56 | - db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN) | ||
| 57 | - if err != nil { | ||
| 58 | - return nil, err | ||
| 59 | - } | ||
| 60 | - db.SetMaxOpenConns(10) | ||
| 61 | - db.SetMaxIdleConns(5) | ||
| 62 | - db.SetConnMaxLifetime(30 * time.Minute) | ||
| 63 | - d := &DB{db} | ||
| 64 | - | ||
| 65 | - if err := initUsersSchema(d); err != nil { | ||
| 66 | - d.Close() | ||
| 67 | - return nil, err | ||
| 68 | - } | ||
| 69 | - | ||
| 70 | - if err := initArticlesSchema(d); err != nil { | ||
| 71 | - d.Close() | ||
| 72 | - return nil, err | ||
| 73 | - } | ||
| 74 | - | ||
| 75 | - if err := initRecsSchema(d); err != nil { | ||
| 76 | - d.Close() | ||
| 77 | - return nil, err | ||
| 78 | - } | ||
| 79 | - | ||
| 80 | - return &Databases{ | ||
| 81 | - Users: NewUserStore(d), | ||
| 82 | - Articles: NewArticleStore(d), | ||
| 83 | - db: d, | ||
| 84 | - }, nil | ||
| 85 | -} | ||
| 86 | - | ||
| 87 | -func (d *Databases) Close() error { | ||
| 88 | - if d.db != nil { | ||
| 89 | - _ = d.db.Close() | ||
| 90 | - } | ||
| 91 | - return nil | ||
| 92 | -} | ||
| 93 | - | ||
| 94 | -func (d *Databases) DB() *sql.DB { | ||
| 95 | - return d.db.DB | ||
| 96 | -} | ||
| 97 | - | ||
| 98 | -func initUsersSchema(db *DB) error { | ||
| 99 | - for _, s := range usersSchema { | ||
| 100 | - if _, err := db.Exec(s); err != nil { | ||
| 101 | - return err | ||
| 102 | - } | ||
| 103 | - } | ||
| 104 | - return nil | ||
| 105 | -} | ||
| 106 | - | ||
| 107 | -func initArticlesSchema(db *DB) error { | ||
| 108 | - for _, s := range articlesSchema { | ||
| 109 | - if _, err := db.Exec(s); err != nil { | ||
| 110 | - return err | ||
| 111 | - } | ||
| 112 | - } | ||
| 113 | - return nil | ||
| 114 | -} | ||
| 115 | - | ||
| 116 | -func initRecsSchema(db *DB) error { | ||
| 117 | - for _, s := range recsSchema { | ||
| 118 | - if _, err := db.Exec(s); err != nil { | ||
| 119 | - return err | ||
| 120 | - } | ||
| 121 | - } | ||
| 122 | - return nil | ||
| 123 | -} | ||
| 124 | - | ||
| 125 | -var usersSchema = []string{ | ||
| 126 | - `CREATE TABLE IF NOT EXISTS users ( | ||
| 127 | - did TEXT PRIMARY KEY, | ||
| 128 | - indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 129 | - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | ||
| 130 | - )`, | ||
| 131 | - | ||
| 132 | - `CREATE TABLE IF NOT EXISTS follows ( | ||
| 133 | - user_did TEXT NOT NULL, | ||
| 134 | - target_did TEXT NOT NULL, | ||
| 135 | - uri TEXT, | ||
| 136 | - cid TEXT, | ||
| 137 | - followed_at DATETIME, | ||
| 138 | - PRIMARY KEY (user_did, target_did) | ||
| 139 | - )`, | ||
| 140 | - | ||
| 141 | - `CREATE TABLE IF NOT EXISTS oauth_auth_requests ( | ||
| 142 | - state TEXT PRIMARY KEY, | ||
| 143 | - data TEXT NOT NULL | ||
| 144 | - )`, | ||
| 145 | - | ||
| 146 | - `CREATE TABLE IF NOT EXISTS oauth_sessions ( | ||
| 147 | - account_did TEXT NOT NULL, | ||
| 148 | - session_id TEXT NOT NULL, | ||
| 149 | - data TEXT NOT NULL, | ||
| 150 | - PRIMARY KEY (account_did, session_id) | ||
| 151 | - )`, | ||
| 152 | - | ||
| 153 | - `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`, | ||
| 154 | - `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`, | ||
| 155 | - `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`, | ||
| 156 | - `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`, | ||
| 157 | -} | ||
| 158 | - | ||
| 159 | -var articlesSchema = []string{ | ||
| 160 | - `CREATE TABLE IF NOT EXISTS articles.feeds ( | ||
| 161 | - feed_url TEXT PRIMARY KEY, | ||
| 162 | - title TEXT, | ||
| 163 | - site_url TEXT, | ||
| 164 | - description TEXT, | ||
| 165 | - feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')), | ||
| 166 | - last_fetched_at DATETIME, | ||
| 167 | - last_error TEXT, | ||
| 168 | - subscriber_count INTEGER NOT NULL DEFAULT 0, | ||
| 169 | - etag TEXT, | ||
| 170 | - last_modified TEXT, | ||
| 171 | - consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0, | ||
| 172 | - error_count INTEGER NOT NULL DEFAULT 0, | ||
| 173 | - favicon_url TEXT | ||
| 174 | - )`, | ||
| 175 | - | ||
| 176 | - `CREATE TABLE IF NOT EXISTS articles.subscriptions ( | ||
| 177 | - id INTEGER PRIMARY KEY AUTOINCREMENT, | ||
| 178 | - user_did TEXT NOT NULL, | ||
| 179 | - feed_url TEXT NOT NULL, | ||
| 180 | - title TEXT, | ||
| 181 | - category TEXT, | ||
| 182 | - added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 183 | - uri TEXT, | ||
| 184 | - cid TEXT, | ||
| 185 | - UNIQUE(user_did, feed_url) | ||
| 186 | - )`, | ||
| 187 | - | ||
| 188 | - `CREATE TABLE IF NOT EXISTS articles.articles ( | ||
| 189 | - id INTEGER PRIMARY KEY AUTOINCREMENT, | ||
| 190 | - feed_url TEXT NOT NULL, | ||
| 191 | - guid TEXT NOT NULL, | ||
| 192 | - title TEXT NOT NULL DEFAULT '', | ||
| 193 | - url TEXT, | ||
| 194 | - author TEXT, | ||
| 195 | - summary TEXT, | ||
| 196 | - content TEXT, | ||
| 197 | - full_content TEXT, | ||
| 198 | - published DATETIME, | ||
| 199 | - updated DATETIME, | ||
| 200 | - fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 201 | - UNIQUE(feed_url, guid) | ||
| 202 | - )`, | ||
| 203 | - | ||
| 204 | - `CREATE TABLE IF NOT EXISTS articles.read_state ( | ||
| 205 | - user_did TEXT NOT NULL, | ||
| 206 | - article_id INTEGER NOT NULL, | ||
| 207 | - is_read BOOLEAN NOT NULL DEFAULT 0, | ||
| 208 | - read_at DATETIME, | ||
| 209 | - PRIMARY KEY (user_did, article_id) | ||
| 210 | - )`, | ||
| 211 | - | ||
| 212 | - `CREATE TABLE IF NOT EXISTS articles.annotations ( | ||
| 213 | - id INTEGER PRIMARY KEY AUTOINCREMENT, | ||
| 214 | - uri TEXT NOT NULL UNIQUE, | ||
| 215 | - author_did TEXT NOT NULL, | ||
| 216 | - feed_url TEXT NOT NULL, | ||
| 217 | - article_url TEXT NOT NULL, | ||
| 218 | - quote TEXT, | ||
| 219 | - note TEXT, | ||
| 220 | - tags TEXT, | ||
| 221 | - rating INTEGER, | ||
| 222 | - created_at DATETIME NOT NULL, | ||
| 223 | - cid TEXT | ||
| 224 | - )`, | ||
| 225 | - | ||
| 226 | - `CREATE TABLE IF NOT EXISTS articles.likes ( | ||
| 227 | - id INTEGER PRIMARY KEY AUTOINCREMENT, | ||
| 228 | - uri TEXT NOT NULL UNIQUE, | ||
| 229 | - author_did TEXT NOT NULL, | ||
| 230 | - feed_url TEXT NOT NULL, | ||
| 231 | - article_url TEXT NOT NULL, | ||
| 232 | - created_at DATETIME NOT NULL, | ||
| 233 | - cid TEXT, | ||
| 234 | - UNIQUE(author_did, feed_url, article_url) | ||
| 235 | - )`, | ||
| 236 | - | ||
| 237 | - `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`, | ||
| 238 | - `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`, | ||
| 239 | - `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`, | ||
| 240 | - `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`, | ||
| 241 | - `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`, | ||
| 242 | - `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`, | ||
| 243 | - `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`, | ||
| 244 | - `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`, | ||
| 245 | - `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`, | ||
| 246 | - `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`, | ||
| 247 | - `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`, | ||
| 248 | - `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`, | ||
| 249 | - `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`, | ||
| 250 | - `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`, | ||
| 251 | - `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`, | ||
| 252 | - | ||
| 253 | - `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`, | ||
| 254 | - `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN | ||
| 255 | - INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author); | ||
| 256 | - END`, | ||
| 257 | - `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN | ||
| 258 | - INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author); | ||
| 259 | - END`, | ||
| 260 | - `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN | ||
| 261 | - INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author); | ||
| 262 | - INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author); | ||
| 263 | - END`, | ||
| 264 | -} | ||
| 265 | - | ||
| 266 | -var recsSchema = []string{ | ||
| 267 | - `CREATE TABLE IF NOT EXISTS recs.feed_similarity ( | ||
| 268 | - feed_a TEXT NOT NULL, | ||
| 269 | - feed_b TEXT NOT NULL, | ||
| 270 | - jaccard REAL NOT NULL, | ||
| 271 | - computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 272 | - PRIMARY KEY (feed_a, feed_b), | ||
| 273 | - CHECK(feed_a < feed_b) | ||
| 274 | - )`, | ||
| 275 | - | ||
| 276 | - `CREATE TABLE IF NOT EXISTS recs.user_similarity ( | ||
| 277 | - user_a TEXT NOT NULL, | ||
| 278 | - user_b TEXT NOT NULL, | ||
| 279 | - jaccard REAL NOT NULL, | ||
| 280 | - common_feeds INTEGER NOT NULL, | ||
| 281 | - common_likes INTEGER NOT NULL DEFAULT 0, | ||
| 282 | - common_tags INTEGER NOT NULL DEFAULT 0, | ||
| 283 | - computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 284 | - PRIMARY KEY (user_a, user_b), | ||
| 285 | - CHECK(user_a < user_b) | ||
| 286 | - )`, | ||
| 287 | - | ||
| 288 | - `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations ( | ||
| 289 | - user_did TEXT NOT NULL, | ||
| 290 | - target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')), | ||
| 291 | - target_id TEXT NOT NULL, | ||
| 292 | - reason TEXT, | ||
| 293 | - dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 294 | - PRIMARY KEY (user_did, target_type, target_id) | ||
| 295 | - )`, | ||
| 296 | - | ||
| 297 | - `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions ( | ||
| 298 | - user_did TEXT NOT NULL, | ||
| 299 | - target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')), | ||
| 300 | - target_id TEXT NOT NULL, | ||
| 301 | - first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 302 | - last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| 303 | - shown_count INTEGER NOT NULL DEFAULT 1, | ||
| 304 | - acted BOOLEAN NOT NULL DEFAULT 0, | ||
| 305 | - PRIMARY KEY (user_did, target_type, target_id) | ||
| 306 | - )`, | ||
| 307 | - | ||
| 308 | - `CREATE TABLE IF NOT EXISTS recs.follow_distances ( | ||
| 309 | - user_a TEXT NOT NULL, | ||
| 310 | - user_b TEXT NOT NULL, | ||
| 311 | - distance INTEGER NOT NULL CHECK(distance IN (1, 2)), | ||
| 312 | - PRIMARY KEY (user_a, user_b) | ||
| 313 | - )`, | ||
| 314 | - | ||
| 315 | - `CREATE TABLE IF NOT EXISTS recs.user_signal_weights ( | ||
| 316 | - user_did TEXT PRIMARY KEY, | ||
| 317 | - w_sub REAL NOT NULL DEFAULT 1.0, | ||
| 318 | - w_like REAL NOT NULL DEFAULT 0.5, | ||
| 319 | - w_tag REAL NOT NULL DEFAULT 0.3, | ||
| 320 | - w_social REAL NOT NULL DEFAULT 0.7, | ||
| 321 | - w_pop REAL NOT NULL DEFAULT 0.2, | ||
| 322 | - w_category REAL NOT NULL DEFAULT 0.4, | ||
| 323 | - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | ||
| 324 | - )`, | ||
| 325 | - | ||
| 326 | - `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles ( | ||
| 327 | - user_did TEXT PRIMARY KEY, | ||
| 328 | - total_likes INTEGER NOT NULL DEFAULT 0, | ||
| 329 | - total_tags INTEGER NOT NULL DEFAULT 0, | ||
| 330 | - top_categories TEXT, | ||
| 331 | - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP | ||
| 332 | - )`, | ||
| 333 | - | ||
| 334 | - `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`, | ||
| 335 | - `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`, | ||
| 336 | - `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`, | ||
| 337 | - `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`, | ||
| 338 | - `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`, | ||
| 339 | - `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`, | ||
| 340 | - `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`, | ||
| 341 | -} | ||
modified
internal/db/social.go +1 -1 | @@ -240,7 +240,7 @@ func (s *ArticleStore) GetLikeCount(ctx context.Context, feedURL, articleURL str | ||
| 240 | 240 | func (s *ArticleStore) GetLike(ctx context.Context, authorDID, feedURL, articleURL string) (*Like, error) { |
| 241 | 241 | l := &Like{} |
| 242 | 242 | err := s.db.QueryRowContext(ctx, ` |
| 243 | - SELECT id, uri, author_did, feed_url, article_url, created_at, cid FROM articles.likes | |
| 243 | + SELECT * FROM articles.likes | |
| 244 | 244 | WHERE author_did = ? AND feed_url = ? AND article_url = ? |
| 245 | 245 | `, authorDID, feedURL, articleURL).Scan(&l.ID, &l.URI, &l.AuthorDID, &l.FeedURL, &l.ArticleURL, &l.CreatedAt, &l.CID) |
| 246 | 246 | if err != nil { |
| @@ -240,7 +240,7 @@ func (s *ArticleStore) GetLikeCount(ctx context.Context, feedURL, articleURL str | |||
| 240 | func (s *ArticleStore) GetLike(ctx context.Context, authorDID, feedURL, articleURL string) (*Like, error) { | 240 | func (s *ArticleStore) GetLike(ctx context.Context, authorDID, feedURL, articleURL string) (*Like, error) { |
| 241 | l := &Like{} | 241 | l := &Like{} |
| 242 | err := s.db.QueryRowContext(ctx, ` | 242 | err := s.db.QueryRowContext(ctx, ` |
| 243 | - SELECT id, uri, author_did, feed_url, article_url, created_at, cid FROM articles.likes | 243 | + SELECT * FROM articles.likes |
| 244 | WHERE author_did = ? AND feed_url = ? AND article_url = ? | 244 | WHERE author_did = ? AND feed_url = ? AND article_url = ? |
| 245 | `, authorDID, feedURL, articleURL).Scan(&l.ID, &l.URI, &l.AuthorDID, &l.FeedURL, &l.ArticleURL, &l.CreatedAt, &l.CID) | 245 | `, authorDID, feedURL, articleURL).Scan(&l.ID, &l.URI, &l.AuthorDID, &l.FeedURL, &l.ArticleURL, &l.CreatedAt, &l.CID) |
| 246 | if err != nil { | 246 | if err != nil { |
modified
internal/db/user.go +1 -2 | @@ -60,8 +60,7 @@ func (s *UserStore) CreateUser(ctx context.Context, did string) (*User, error) { | ||
| 60 | 60 | func (s *UserStore) GetUser(ctx context.Context, did string) (*User, error) { |
| 61 | 61 | u := &User{} |
| 62 | 62 | err := s.db.QueryRowContext(ctx, ` |
| 63 | - SELECT did, indexed_at, updated_at | |
| 64 | - FROM users WHERE did = ? | |
| 63 | + SELECT * FROM users WHERE did = ? | |
| 65 | 64 | `, did).Scan(&u.DID, &u.IndexedAt, &u.UpdatedAt) |
| 66 | 65 | if err != nil { |
| 67 | 66 | return nil, err |
| @@ -60,8 +60,7 @@ func (s *UserStore) CreateUser(ctx context.Context, did string) (*User, error) { | |||
| 60 | func (s *UserStore) GetUser(ctx context.Context, did string) (*User, error) { | 60 | func (s *UserStore) GetUser(ctx context.Context, did string) (*User, error) { |
| 61 | u := &User{} | 61 | u := &User{} |
| 62 | err := s.db.QueryRowContext(ctx, ` | 62 | err := s.db.QueryRowContext(ctx, ` |
| 63 | - SELECT did, indexed_at, updated_at | 63 | + SELECT * FROM users WHERE did = ? |
| 64 | - FROM users WHERE did = ? | ||
| 65 | `, did).Scan(&u.DID, &u.IndexedAt, &u.UpdatedAt) | 64 | `, did).Scan(&u.DID, &u.IndexedAt, &u.UpdatedAt) |
| 66 | if err != nil { | 65 | if err != nil { |
| 67 | return nil, err | 66 | return nil, err |
modified
internal/feed/discover.go +2 -2 | @@ -130,7 +130,7 @@ func findFavicon(ctx context.Context, base *url.URL, links []string) string { | ||
| 130 | 130 | if err != nil { |
| 131 | 131 | return |
| 132 | 132 | } |
| 133 | - resp.Body.Close() | |
| 133 | + _ = resp.Body.Close() | |
| 134 | 134 | if resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type")) { |
| 135 | 135 | select { |
| 136 | 136 | case found <- result{url: cleanFavicon(resolved.String()), found: true}: |
| @@ -160,7 +160,7 @@ func checkContentType(ctx context.Context, url string) bool { | ||
| 160 | 160 | if err != nil { |
| 161 | 161 | return false |
| 162 | 162 | } |
| 163 | - resp.Body.Close() | |
| 163 | + _ = resp.Body.Close() | |
| 164 | 164 | return resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type")) |
| 165 | 165 | } |
| 166 | 166 | |
| @@ -130,7 +130,7 @@ func findFavicon(ctx context.Context, base *url.URL, links []string) string { | |||
| 130 | if err != nil { | 130 | if err != nil { |
| 131 | return | 131 | return |
| 132 | } | 132 | } |
| 133 | - resp.Body.Close() | 133 | + _ = resp.Body.Close() |
| 134 | if resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type")) { | 134 | if resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type")) { |
| 135 | select { | 135 | select { |
| 136 | case found <- result{url: cleanFavicon(resolved.String()), found: true}: | 136 | case found <- result{url: cleanFavicon(resolved.String()), found: true}: |
| @@ -160,7 +160,7 @@ func checkContentType(ctx context.Context, url string) bool { | |||
| 160 | if err != nil { | 160 | if err != nil { |
| 161 | return false | 161 | return false |
| 162 | } | 162 | } |
| 163 | - resp.Body.Close() | 163 | + _ = resp.Body.Close() |
| 164 | return resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type")) | 164 | return resp.StatusCode == http.StatusOK && isImageContentType(resp.Header.Get("Content-Type")) |
| 165 | } | 165 | } |
| 166 | 166 | ||
modified
internal/feed/fetcher.go +1 -1 | @@ -221,7 +221,7 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) { | ||
| 221 | 221 | metrics.FeedsFetchedLast.Set(float64(time.Now().Unix())) |
| 222 | 222 | if err != nil { |
| 223 | 223 | s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL) |
| 224 | - s.store.RecordFetchError(ctx, feed.URL, err.Error()) | |
| 224 | + _ = s.store.RecordFetchError(ctx, feed.URL, err.Error()) | |
| 225 | 225 | return |
| 226 | 226 | } |
| 227 | 227 | |
| @@ -221,7 +221,7 @@ func (s *Scheduler) FetchFeed(ctx context.Context, feed *Feed) { | |||
| 221 | metrics.FeedsFetchedLast.Set(float64(time.Now().Unix())) | 221 | metrics.FeedsFetchedLast.Set(float64(time.Now().Unix())) |
| 222 | if err != nil { | 222 | if err != nil { |
| 223 | s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL) | 223 | s.logger.Error("failed to fetch feed", "error", err, "feed", feed.URL) |
| 224 | - s.store.RecordFetchError(ctx, feed.URL, err.Error()) | 224 | + _ = s.store.RecordFetchError(ctx, feed.URL, err.Error()) |
| 225 | return | 225 | return |
| 226 | } | 226 | } |
| 227 | 227 | ||
modified
internal/httpclient/httpclient.go +1 -1 | @@ -19,7 +19,7 @@ const ( | ||
| 19 | 19 | |
| 20 | 20 | var dnsResolver = &net.Resolver{ |
| 21 | 21 | PreferGo: true, |
| 22 | - Dial: func(ctx context.Context, network, address string) (net.Conn, error) { | |
| 22 | + Dial: func(ctx context.Context, _, address string) (net.Conn, error) { | |
| 23 | 23 | d := net.Dialer{Timeout: 3 * time.Second} |
| 24 | 24 | return d.DialContext(ctx, "udp", "1.1.1.1:53") |
| 25 | 25 | }, |
| @@ -19,7 +19,7 @@ const ( | |||
| 19 | 19 | ||
| 20 | var dnsResolver = &net.Resolver{ | 20 | var dnsResolver = &net.Resolver{ |
| 21 | PreferGo: true, | 21 | PreferGo: true, |
| 22 | - Dial: func(ctx context.Context, network, address string) (net.Conn, error) { | 22 | + Dial: func(ctx context.Context, _, address string) (net.Conn, error) { |
| 23 | d := net.Dialer{Timeout: 3 * time.Second} | 23 | d := net.Dialer{Timeout: 3 * time.Second} |
| 24 | return d.DialContext(ctx, "udp", "1.1.1.1:53") | 24 | return d.DialContext(ctx, "udp", "1.1.1.1:53") |
| 25 | }, | 25 | }, |
modified
internal/server/articles_handler.go +1 -1 | @@ -104,7 +104,7 @@ func (s *Server) handleArticles(w http.ResponseWriter, r *http.Request) { | ||
| 104 | 104 | } |
| 105 | 105 | } |
| 106 | 106 | |
| 107 | - if r.Header.Get("HX-Request") == "true" { | |
| 107 | + if r.Header.Get("HX-Request") == htmxRequestHeader { | |
| 108 | 108 | s.render(w, r, "articles-content.html", data) |
| 109 | 109 | return |
| 110 | 110 | } |
| @@ -104,7 +104,7 @@ func (s *Server) handleArticles(w http.ResponseWriter, r *http.Request) { | |||
| 104 | } | 104 | } |
| 105 | } | 105 | } |
| 106 | 106 | ||
| 107 | - if r.Header.Get("HX-Request") == "true" { | 107 | + if r.Header.Get("HX-Request") == htmxRequestHeader { |
| 108 | s.render(w, r, "articles-content.html", data) | 108 | s.render(w, r, "articles-content.html", data) |
| 109 | return | 109 | return |
| 110 | } | 110 | } |
modified
internal/server/middleware.go +1 -1 | @@ -55,7 +55,7 @@ func (s *Server) csrfMiddleware(next http.Handler) http.Handler { | ||
| 55 | 55 | return |
| 56 | 56 | } |
| 57 | 57 | |
| 58 | - if r.Header.Get("HX-Request") == "true" { | |
| 58 | + if r.Header.Get("HX-Request") == htmxRequestHeader { | |
| 59 | 59 | origin := r.Header.Get("Origin") |
| 60 | 60 | if origin != "" && !sameOrigin(origin, r.Host) { |
| 61 | 61 | http.Error(w, "forbidden", http.StatusForbidden) |
| @@ -55,7 +55,7 @@ func (s *Server) csrfMiddleware(next http.Handler) http.Handler { | |||
| 55 | return | 55 | return |
| 56 | } | 56 | } |
| 57 | 57 | ||
| 58 | - if r.Header.Get("HX-Request") == "true" { | 58 | + if r.Header.Get("HX-Request") == htmxRequestHeader { |
| 59 | origin := r.Header.Get("Origin") | 59 | origin := r.Header.Get("Origin") |
| 60 | if origin != "" && !sameOrigin(origin, r.Host) { | 60 | if origin != "" && !sameOrigin(origin, r.Host) { |
| 61 | http.Error(w, "forbidden", http.StatusForbidden) | 61 | http.Error(w, "forbidden", http.StatusForbidden) |
modified
internal/server/server.go +6 -8 | @@ -33,6 +33,8 @@ import ( | ||
| 33 | 33 | "pkg.rbrt.fr/glean/static" |
| 34 | 34 | ) |
| 35 | 35 | |
| 36 | +const htmxRequestHeader = "true" | |
| 37 | + | |
| 36 | 38 | var oauthScopes = []string{ |
| 37 | 39 | "atproto", |
| 38 | 40 | "blob:*/*", |
| @@ -314,12 +316,8 @@ func (s *Server) loadTemplates() { | ||
| 314 | 316 | "sanitizeHTML": func(input string) template.HTML { |
| 315 | 317 | return template.HTML(sanitize.HTML(input)) |
| 316 | 318 | }, |
| 317 | - "plainText": func(input string) string { | |
| 318 | - return sanitize.PlainText(input) | |
| 319 | - }, | |
| 320 | - "now": func() time.Time { | |
| 321 | - return time.Now() | |
| 322 | - }, | |
| 319 | + "plainText": sanitize.PlainText, | |
| 320 | + "now": time.Now, | |
| 323 | 321 | "activeClass": func(activePath, linkPath string) string { |
| 324 | 322 | if activePath == linkPath || (len(activePath) > len(linkPath) && activePath[:len(linkPath)+1] == linkPath+"/") { |
| 325 | 323 | return "bg-spot-hover text-spot-text font-bold" |
| @@ -515,7 +513,7 @@ func (s *Server) handleNotFound(w http.ResponseWriter, r *http.Request) { | ||
| 515 | 513 | } |
| 516 | 514 | |
| 517 | 515 | func (s *Server) renderError(w http.ResponseWriter, r *http.Request, code int, title, message string) { |
| 518 | - if r.Header.Get("HX-Request") == "true" { | |
| 516 | + if r.Header.Get("HX-Request") == htmxRequestHeader { | |
| 519 | 517 | w.WriteHeader(code) |
| 520 | 518 | w.Write([]byte(message)) |
| 521 | 519 | return |
| @@ -536,7 +534,7 @@ func (s *Server) render(w http.ResponseWriter, r *http.Request, name string, dat | ||
| 536 | 534 | data["CSRFToken"] = cookie.Value |
| 537 | 535 | } |
| 538 | 536 | |
| 539 | - if r.Header.Get("HX-Request") == "true" { | |
| 537 | + if r.Header.Get("HX-Request") == htmxRequestHeader { | |
| 540 | 538 | if err := s.templates.ExecuteTemplate(w, name, data); err != nil { |
| 541 | 539 | s.logger.Error("template error", "error", err, "template", name) |
| 542 | 540 | http.Error(w, err.Error(), http.StatusInternalServerError) |
| @@ -33,6 +33,8 @@ import ( | |||
| 33 | "pkg.rbrt.fr/glean/static" | 33 | "pkg.rbrt.fr/glean/static" |
| 34 | ) | 34 | ) |
| 35 | 35 | ||
| 36 | +const htmxRequestHeader = "true" | ||
| 37 | + | ||
| 36 | var oauthScopes = []string{ | 38 | var oauthScopes = []string{ |
| 37 | "atproto", | 39 | "atproto", |
| 38 | "blob:*/*", | 40 | "blob:*/*", |
| @@ -314,12 +316,8 @@ func (s *Server) loadTemplates() { | |||
| 314 | "sanitizeHTML": func(input string) template.HTML { | 316 | "sanitizeHTML": func(input string) template.HTML { |
| 315 | return template.HTML(sanitize.HTML(input)) | 317 | return template.HTML(sanitize.HTML(input)) |
| 316 | }, | 318 | }, |
| 317 | - "plainText": func(input string) string { | 319 | + "plainText": sanitize.PlainText, |
| 318 | - return sanitize.PlainText(input) | 320 | + "now": time.Now, |
| 319 | - }, | ||
| 320 | - "now": func() time.Time { | ||
| 321 | - return time.Now() | ||
| 322 | - }, | ||
| 323 | "activeClass": func(activePath, linkPath string) string { | 321 | "activeClass": func(activePath, linkPath string) string { |
| 324 | if activePath == linkPath || (len(activePath) > len(linkPath) && activePath[:len(linkPath)+1] == linkPath+"/") { | 322 | if activePath == linkPath || (len(activePath) > len(linkPath) && activePath[:len(linkPath)+1] == linkPath+"/") { |
| 325 | return "bg-spot-hover text-spot-text font-bold" | 323 | return "bg-spot-hover text-spot-text font-bold" |
| @@ -515,7 +513,7 @@ func (s *Server) handleNotFound(w http.ResponseWriter, r *http.Request) { | |||
| 515 | } | 513 | } |
| 516 | 514 | ||
| 517 | func (s *Server) renderError(w http.ResponseWriter, r *http.Request, code int, title, message string) { | 515 | func (s *Server) renderError(w http.ResponseWriter, r *http.Request, code int, title, message string) { |
| 518 | - if r.Header.Get("HX-Request") == "true" { | 516 | + if r.Header.Get("HX-Request") == htmxRequestHeader { |
| 519 | w.WriteHeader(code) | 517 | w.WriteHeader(code) |
| 520 | w.Write([]byte(message)) | 518 | w.Write([]byte(message)) |
| 521 | return | 519 | return |
| @@ -536,7 +534,7 @@ func (s *Server) render(w http.ResponseWriter, r *http.Request, name string, dat | |||
| 536 | data["CSRFToken"] = cookie.Value | 534 | data["CSRFToken"] = cookie.Value |
| 537 | } | 535 | } |
| 538 | 536 | ||
| 539 | - if r.Header.Get("HX-Request") == "true" { | 537 | + if r.Header.Get("HX-Request") == htmxRequestHeader { |
| 540 | if err := s.templates.ExecuteTemplate(w, name, data); err != nil { | 538 | if err := s.templates.ExecuteTemplate(w, name, data); err != nil { |
| 541 | s.logger.Error("template error", "error", err, "template", name) | 539 | s.logger.Error("template error", "error", err, "template", name) |
| 542 | http.Error(w, err.Error(), http.StatusInternalServerError) | 540 | http.Error(w, err.Error(), http.StatusInternalServerError) |
modified
internal/server/trending_handler.go +4 -2 | @@ -7,16 +7,18 @@ import ( | ||
| 7 | 7 | "pkg.rbrt.fr/glean/internal/db" |
| 8 | 8 | ) |
| 9 | 9 | |
| 10 | +const scopeForMe = "for-me" | |
| 11 | + | |
| 10 | 12 | func (s *Server) handleTrending(w http.ResponseWriter, r *http.Request) { |
| 11 | 13 | user := currentUser(r) |
| 12 | 14 | ctx := r.Context() |
| 13 | 15 | |
| 14 | 16 | scope := r.URL.Query().Get("scope") |
| 15 | - if scope == "for-me" && user == nil { | |
| 17 | + if scope == scopeForMe && user == nil { | |
| 16 | 18 | http.Redirect(w, r, "/auth/login", http.StatusSeeOther) |
| 17 | 19 | return |
| 18 | 20 | } |
| 19 | - if scope != "for-me" { | |
| 21 | + if scope != scopeForMe { | |
| 20 | 22 | scope = "all" |
| 21 | 23 | } |
| 22 | 24 | |
| @@ -7,16 +7,18 @@ import ( | |||
| 7 | "pkg.rbrt.fr/glean/internal/db" | 7 | "pkg.rbrt.fr/glean/internal/db" |
| 8 | ) | 8 | ) |
| 9 | 9 | ||
| 10 | +const scopeForMe = "for-me" | ||
| 11 | + | ||
| 10 | func (s *Server) handleTrending(w http.ResponseWriter, r *http.Request) { | 12 | func (s *Server) handleTrending(w http.ResponseWriter, r *http.Request) { |
| 11 | user := currentUser(r) | 13 | user := currentUser(r) |
| 12 | ctx := r.Context() | 14 | ctx := r.Context() |
| 13 | 15 | ||
| 14 | scope := r.URL.Query().Get("scope") | 16 | scope := r.URL.Query().Get("scope") |
| 15 | - if scope == "for-me" && user == nil { | 17 | + if scope == scopeForMe && user == nil { |
| 16 | http.Redirect(w, r, "/auth/login", http.StatusSeeOther) | 18 | http.Redirect(w, r, "/auth/login", http.StatusSeeOther) |
| 17 | return | 19 | return |
| 18 | } | 20 | } |
| 19 | - if scope != "for-me" { | 21 | + if scope != scopeForMe { |
| 20 | scope = "all" | 22 | scope = "all" |
| 21 | } | 23 | } |
| 22 | 24 | ||