rickub
nandi/gleanpublic⑂ Fork 0
⑂ aa5c138
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Support standard.site publications as "RSS"Unverified

Julien Robert committed 2026-05-01T21:46:01+02:00 Browse files
aa5c138 parent: 56b717c
modified internal/atproto/client.go +21 -0
@@ -100,6 +100,27 @@ func (c *Client) ListRecords(ctx context.Context, did, collection string, limit
100100 return records, result.Cursor, nil
101101 }
102102
103+func (c *Client) GetRecord(ctx context.Context, did, collection, rkey string) (json.RawMessage, error) {
104+ nsid, err := syntax.ParseNSID("com.atproto.repo.getRecord")
105+ if err != nil {
106+ return nil, fmt.Errorf("parsing NSID: %w", err)
107+ }
108+
109+ params := map[string]any{
110+ "repo": did,
111+ "collection": collection,
112+ "rkey": rkey,
113+ }
114+
115+ var result struct {
116+ Value json.RawMessage `json:"value"`
117+ }
118+ if err := c.api.Get(ctx, nsid, params, &result); err != nil {
119+ return nil, err
120+ }
121+ return result.Value, nil
122+}
123+
103124 func (c *Client) GetProfile(ctx context.Context, did string) (displayName, avatarURL string, err error) {
104125 nsid, err := syntax.ParseNSID("app.bsky.actor.getProfile")
105126 if err != nil {
@@ -100,6 +100,27 @@ func (c *Client) ListRecords(ctx context.Context, did, collection string, limit
100 return records, result.Cursor, nil100 return records, result.Cursor, nil
101 }101 }
102 102
103+func (c *Client) GetRecord(ctx context.Context, did, collection, rkey string) (json.RawMessage, error) {
104+ nsid, err := syntax.ParseNSID("com.atproto.repo.getRecord")
105+ if err != nil {
106+ return nil, fmt.Errorf("parsing NSID: %w", err)
107+ }
108+
109+ params := map[string]any{
110+ "repo": did,
111+ "collection": collection,
112+ "rkey": rkey,
113+ }
114+
115+ var result struct {
116+ Value json.RawMessage `json:"value"`
117+ }
118+ if err := c.api.Get(ctx, nsid, params, &result); err != nil {
119+ return nil, err
120+ }
121+ return result.Value, nil
122+}
123+
103 func (c *Client) GetProfile(ctx context.Context, did string) (displayName, avatarURL string, err error) {124 func (c *Client) GetProfile(ctx context.Context, did string) (displayName, avatarURL string, err error) {
104 nsid, err := syntax.ParseNSID("app.bsky.actor.getProfile")125 nsid, err := syntax.ParseNSID("app.bsky.actor.getProfile")
105 if err != nil {126 if err != nil {
modified internal/atproto/jetstream.go +2 -0
@@ -100,6 +100,8 @@ func NewJetstreamConsumer(jetstreamURL string, handler EventHandler, logger *slo
100100 CollectionBskyFollow,
101101 CollectionTangledFollow,
102102 CollectionMarginNote,
103+ CollectionStandardPublication,
104+ CollectionStandardDocument,
103105 },
104106 }
105107
@@ -100,6 +100,8 @@ func NewJetstreamConsumer(jetstreamURL string, handler EventHandler, logger *slo
100 CollectionBskyFollow,100 CollectionBskyFollow,
101 CollectionTangledFollow,101 CollectionTangledFollow,
102 CollectionMarginNote,102 CollectionMarginNote,
103+ CollectionStandardPublication,
104+ CollectionStandardDocument,
103 },105 },
104 }106 }
105 107
modified internal/atproto/lexicon.go +2 -0
@@ -15,6 +15,8 @@ const (
1515 CollectionSkyreaderSubscription = "app.skyreader.feed.subscription"
1616 CollectionBskyFollow = "app.bsky.graph.follow"
1717 CollectionTangledFollow = "sh.tangled.graph.follow"
18+ CollectionStandardPublication = "site.standard.publication"
19+ CollectionStandardDocument = "site.standard.document"
1820 )
1921
2022 type SubscriptionRecord struct {
@@ -15,6 +15,8 @@ const (
15 CollectionSkyreaderSubscription = "app.skyreader.feed.subscription"15 CollectionSkyreaderSubscription = "app.skyreader.feed.subscription"
16 CollectionBskyFollow = "app.bsky.graph.follow"16 CollectionBskyFollow = "app.bsky.graph.follow"
17 CollectionTangledFollow = "sh.tangled.graph.follow"17 CollectionTangledFollow = "sh.tangled.graph.follow"
18+ CollectionStandardPublication = "site.standard.publication"
19+ CollectionStandardDocument = "site.standard.document"
18 )20 )
19 21
20 type SubscriptionRecord struct {22 type SubscriptionRecord struct {
modified internal/atproto/lexicon_external.go +27 -0
@@ -89,3 +89,30 @@ func (r MarginNoteRecord) ToAnnotation() (articleURL, quote, note string, tags [
8989 tags = r.Tags
9090 return
9191 }
92+
93+type StandardPublicationRecord struct {
94+ BasicTheme json.RawMessage `json:"basicTheme,omitempty"`
95+ Name string `json:"name"`
96+ URL string `json:"url"`
97+ Description string `json:"description"`
98+ Icon json.RawMessage `json:"icon,omitempty"`
99+ Labels json.RawMessage `json:"labels,omitempty"`
100+ Preferences json.RawMessage `json:"preferences,omitempty"`
101+}
102+
103+type StandardDocumentRecord struct {
104+ Title string `json:"title"`
105+ Site string `json:"site"`
106+ Path string `json:"path"`
107+ Description string `json:"description"`
108+ TextContent string `json:"textContent"`
109+ PublishedAt string `json:"publishedAt"`
110+ UpdatedAt string `json:"updatedAt"`
111+ BskyPostRef json.RawMessage `json:"bskyPostRef,omitempty"`
112+ Content json.RawMessage `json:"content,omitempty"`
113+ Contributors json.RawMessage `json:"contributors,omitempty"`
114+ CoverImage json.RawMessage `json:"coverImage,omitempty"`
115+ Labels json.RawMessage `json:"labels,omitempty"`
116+ Links json.RawMessage `json:"links,omitempty"`
117+ Tags []string `json:"tags,omitempty"`
118+}
@@ -89,3 +89,30 @@ func (r MarginNoteRecord) ToAnnotation() (articleURL, quote, note string, tags [
89 tags = r.Tags89 tags = r.Tags
90 return90 return
91 }91 }
92+
93+type StandardPublicationRecord struct {
94+ BasicTheme json.RawMessage `json:"basicTheme,omitempty"`
95+ Name string `json:"name"`
96+ URL string `json:"url"`
97+ Description string `json:"description"`
98+ Icon json.RawMessage `json:"icon,omitempty"`
99+ Labels json.RawMessage `json:"labels,omitempty"`
100+ Preferences json.RawMessage `json:"preferences,omitempty"`
101+}
102+
103+type StandardDocumentRecord struct {
104+ Title string `json:"title"`
105+ Site string `json:"site"`
106+ Path string `json:"path"`
107+ Description string `json:"description"`
108+ TextContent string `json:"textContent"`
109+ PublishedAt string `json:"publishedAt"`
110+ UpdatedAt string `json:"updatedAt"`
111+ BskyPostRef json.RawMessage `json:"bskyPostRef,omitempty"`
112+ Content json.RawMessage `json:"content,omitempty"`
113+ Contributors json.RawMessage `json:"contributors,omitempty"`
114+ CoverImage json.RawMessage `json:"coverImage,omitempty"`
115+ Labels json.RawMessage `json:"labels,omitempty"`
116+ Links json.RawMessage `json:"links,omitempty"`
117+ Tags []string `json:"tags,omitempty"`
118+}
modified internal/atproto/lexicon_test.go +8 -0
@@ -93,3 +93,11 @@ func TestMarginNoteRecordMatchesLexicon(t *testing.T) {
9393 func TestSkyreaderSubscriptionRecordMatchesLexicon(t *testing.T) {
9494 assertStructMatchesLexiconPath[SkyreaderSubscriptionRecord](t, lexiconPathFromRoot("app/skyreader/feed/subscription.json"))
9595 }
96+
97+func TestStandardPublicationRecordMatchesLexicon(t *testing.T) {
98+ assertStructMatchesLexiconPath[StandardPublicationRecord](t, lexiconPathFromRoot("site/standard/publication.json"))
99+}
100+
101+func TestStandardDocumentRecordMatchesLexicon(t *testing.T) {
102+ assertStructMatchesLexiconPath[StandardDocumentRecord](t, lexiconPathFromRoot("site/standard/document.json"))
103+}
@@ -93,3 +93,11 @@ func TestMarginNoteRecordMatchesLexicon(t *testing.T) {
93 func TestSkyreaderSubscriptionRecordMatchesLexicon(t *testing.T) {93 func TestSkyreaderSubscriptionRecordMatchesLexicon(t *testing.T) {
94 assertStructMatchesLexiconPath[SkyreaderSubscriptionRecord](t, lexiconPathFromRoot("app/skyreader/feed/subscription.json"))94 assertStructMatchesLexiconPath[SkyreaderSubscriptionRecord](t, lexiconPathFromRoot("app/skyreader/feed/subscription.json"))
95 }95 }
96+
97+func TestStandardPublicationRecordMatchesLexicon(t *testing.T) {
98+ assertStructMatchesLexiconPath[StandardPublicationRecord](t, lexiconPathFromRoot("site/standard/publication.json"))
99+}
100+
101+func TestStandardDocumentRecordMatchesLexicon(t *testing.T) {
102+ assertStructMatchesLexiconPath[StandardDocumentRecord](t, lexiconPathFromRoot("site/standard/document.json"))
103+}
added internal/atproto/standard_site_fetcher.go +146 -0
new file mode 100644
@@ -0,0 +1,146 @@
1+package atproto
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "fmt"
7+ "log/slog"
8+ "strings"
9+ "time"
10+
11+ "pkg.rbrt.fr/glean/internal/feed"
12+)
13+
14+func IsATProtoFeedURL(feedURL string) bool {
15+ return strings.HasPrefix(feedURL, "at://")
16+}
17+
18+type StandardSiteFetcher struct {
19+ logger *slog.Logger
20+}
21+
22+func NewStandardSiteFetcher(logger *slog.Logger) *StandardSiteFetcher {
23+ return &StandardSiteFetcher{logger: logger}
24+}
25+
26+func (f *StandardSiteFetcher) FetchFeed(ctx context.Context, feedURL string) (*feed.ParseResult, error) {
27+ parsed, ok := ParseRecordURI(feedURL)
28+ if !ok {
29+ return nil, fmt.Errorf("invalid AT URI: %s", feedURL)
30+ }
31+
32+ if parsed.Collection != CollectionStandardPublication {
33+ return nil, fmt.Errorf("unsupported AT Protocol collection: %s", parsed.Collection)
34+ }
35+
36+ pdsURL, err := ResolvePDSEndpoint(ctx, parsed.DID)
37+ if err != nil {
38+ return nil, fmt.Errorf("resolving PDS for %s: %w", parsed.DID, err)
39+ }
40+
41+ client := NewUnauthenticatedClient(pdsURL)
42+
43+ raw, err := client.GetRecord(ctx, parsed.DID, parsed.Collection, parsed.RKey)
44+ if err != nil {
45+ return nil, fmt.Errorf("fetching publication record: %w", err)
46+ }
47+
48+ var pub StandardPublicationRecord
49+ if err := json.Unmarshal(raw, &pub); err != nil {
50+ return nil, fmt.Errorf("parsing publication record: %w", err)
51+ }
52+
53+ documents, err := f.fetchDocuments(ctx, client, parsed.DID, feedURL, pub.URL)
54+ if err != nil {
55+ f.logger.Warn("failed to fetch documents for publication", "error", err, "uri", feedURL)
56+ }
57+
58+ result := &feed.ParseResult{
59+ Feed: feed.Feed{
60+ URL: feedURL,
61+ Title: pub.Name,
62+ SiteURL: pub.URL,
63+ Description: pub.Description,
64+ Type: "atproto",
65+ },
66+ Articles: documents,
67+ }
68+
69+ return result, nil
70+}
71+
72+func (f *StandardSiteFetcher) fetchDocuments(ctx context.Context, client *Client, did, publicationURI, publicationURL string) ([]feed.Article, error) {
73+ records, err := listAllRecords(ctx, client, did, CollectionStandardDocument)
74+ if err != nil {
75+ return nil, fmt.Errorf("listing documents: %w", err)
76+ }
77+
78+ var articles []feed.Article
79+ for _, r := range records {
80+ var doc StandardDocumentRecord
81+ if err := json.Unmarshal(r.Value, &doc); err != nil {
82+ continue
83+ }
84+ if doc.Title == "" {
85+ continue
86+ }
87+ if doc.Site != publicationURI && doc.Site != publicationURL {
88+ continue
89+ }
90+
91+ published := parseRFC3339(doc.PublishedAt)
92+ updated := parseRFC3339(doc.UpdatedAt)
93+
94+ articles = append(articles, feed.Article{
95+ FeedURL: publicationURI,
96+ GUID: r.URI,
97+ Title: doc.Title,
98+ URL: publicationURL + doc.Path,
99+ Author: firstContributor(doc.Contributors),
100+ Content: doc.TextContent,
101+ Summary: doc.Description,
102+ Published: published,
103+ Updated: updated,
104+ })
105+ }
106+
107+ return articles, nil
108+}
109+
110+func firstContributor(raw json.RawMessage) string {
111+ if len(raw) == 0 {
112+ return ""
113+ }
114+ var contributors []struct {
115+ DisplayName string `json:"displayName"`
116+ }
117+ if err := json.Unmarshal(raw, &contributors); err != nil {
118+ return ""
119+ }
120+ if len(contributors) == 0 || contributors[0].DisplayName == "" {
121+ return ""
122+ }
123+ return contributors[0].DisplayName
124+}
125+
126+func listAllRecords(ctx context.Context, client *Client, did, collection string) ([]Record, error) {
127+ var all []Record
128+ cursor := ""
129+ for {
130+ records, next, err := client.ListRecords(ctx, did, collection, 100, cursor)
131+ if err != nil {
132+ return nil, err
133+ }
134+ all = append(all, records...)
135+ if next == "" || len(records) == 0 {
136+ break
137+ }
138+ cursor = next
139+ }
140+ return all, nil
141+}
142+
143+func parseRFC3339(s string) time.Time {
144+ t, _ := time.Parse(time.RFC3339, s)
145+ return t
146+}
new file mode 100644
@@ -0,0 +1,146 @@
1+package atproto
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "fmt"
7+ "log/slog"
8+ "strings"
9+ "time"
10+
11+ "pkg.rbrt.fr/glean/internal/feed"
12+)
13+
14+func IsATProtoFeedURL(feedURL string) bool {
15+ return strings.HasPrefix(feedURL, "at://")
16+}
17+
18+type StandardSiteFetcher struct {
19+ logger *slog.Logger
20+}
21+
22+func NewStandardSiteFetcher(logger *slog.Logger) *StandardSiteFetcher {
23+ return &StandardSiteFetcher{logger: logger}
24+}
25+
26+func (f *StandardSiteFetcher) FetchFeed(ctx context.Context, feedURL string) (*feed.ParseResult, error) {
27+ parsed, ok := ParseRecordURI(feedURL)
28+ if !ok {
29+ return nil, fmt.Errorf("invalid AT URI: %s", feedURL)
30+ }
31+
32+ if parsed.Collection != CollectionStandardPublication {
33+ return nil, fmt.Errorf("unsupported AT Protocol collection: %s", parsed.Collection)
34+ }
35+
36+ pdsURL, err := ResolvePDSEndpoint(ctx, parsed.DID)
37+ if err != nil {
38+ return nil, fmt.Errorf("resolving PDS for %s: %w", parsed.DID, err)
39+ }
40+
41+ client := NewUnauthenticatedClient(pdsURL)
42+
43+ raw, err := client.GetRecord(ctx, parsed.DID, parsed.Collection, parsed.RKey)
44+ if err != nil {
45+ return nil, fmt.Errorf("fetching publication record: %w", err)
46+ }
47+
48+ var pub StandardPublicationRecord
49+ if err := json.Unmarshal(raw, &pub); err != nil {
50+ return nil, fmt.Errorf("parsing publication record: %w", err)
51+ }
52+
53+ documents, err := f.fetchDocuments(ctx, client, parsed.DID, feedURL, pub.URL)
54+ if err != nil {
55+ f.logger.Warn("failed to fetch documents for publication", "error", err, "uri", feedURL)
56+ }
57+
58+ result := &feed.ParseResult{
59+ Feed: feed.Feed{
60+ URL: feedURL,
61+ Title: pub.Name,
62+ SiteURL: pub.URL,
63+ Description: pub.Description,
64+ Type: "atproto",
65+ },
66+ Articles: documents,
67+ }
68+
69+ return result, nil
70+}
71+
72+func (f *StandardSiteFetcher) fetchDocuments(ctx context.Context, client *Client, did, publicationURI, publicationURL string) ([]feed.Article, error) {
73+ records, err := listAllRecords(ctx, client, did, CollectionStandardDocument)
74+ if err != nil {
75+ return nil, fmt.Errorf("listing documents: %w", err)
76+ }
77+
78+ var articles []feed.Article
79+ for _, r := range records {
80+ var doc StandardDocumentRecord
81+ if err := json.Unmarshal(r.Value, &doc); err != nil {
82+ continue
83+ }
84+ if doc.Title == "" {
85+ continue
86+ }
87+ if doc.Site != publicationURI && doc.Site != publicationURL {
88+ continue
89+ }
90+
91+ published := parseRFC3339(doc.PublishedAt)
92+ updated := parseRFC3339(doc.UpdatedAt)
93+
94+ articles = append(articles, feed.Article{
95+ FeedURL: publicationURI,
96+ GUID: r.URI,
97+ Title: doc.Title,
98+ URL: publicationURL + doc.Path,
99+ Author: firstContributor(doc.Contributors),
100+ Content: doc.TextContent,
101+ Summary: doc.Description,
102+ Published: published,
103+ Updated: updated,
104+ })
105+ }
106+
107+ return articles, nil
108+}
109+
110+func firstContributor(raw json.RawMessage) string {
111+ if len(raw) == 0 {
112+ return ""
113+ }
114+ var contributors []struct {
115+ DisplayName string `json:"displayName"`
116+ }
117+ if err := json.Unmarshal(raw, &contributors); err != nil {
118+ return ""
119+ }
120+ if len(contributors) == 0 || contributors[0].DisplayName == "" {
121+ return ""
122+ }
123+ return contributors[0].DisplayName
124+}
125+
126+func listAllRecords(ctx context.Context, client *Client, did, collection string) ([]Record, error) {
127+ var all []Record
128+ cursor := ""
129+ for {
130+ records, next, err := client.ListRecords(ctx, did, collection, 100, cursor)
131+ if err != nil {
132+ return nil, err
133+ }
134+ all = append(all, records...)
135+ if next == "" || len(records) == 0 {
136+ break
137+ }
138+ cursor = next
139+ }
140+ return all, nil
141+}
142+
143+func parseRFC3339(s string) time.Time {
144+ t, _ := time.Parse(time.RFC3339, s)
145+ return t
146+}
modified internal/atproto/stream_handler.go +55 -0
@@ -9,6 +9,7 @@ import (
99 "time"
1010
1111 "pkg.rbrt.fr/glean/internal/db"
12+ "pkg.rbrt.fr/glean/internal/feed"
1213 )
1314
1415 const (
@@ -52,6 +53,8 @@ func (h *StreamDBHandler) Handle(ctx context.Context, event *Event) error {
5253 return h.handleMarginNote(ctx, event)
5354 case CollectionBskyFollow, CollectionTangledFollow:
5455 return h.handleFollow(ctx, event)
56+ case CollectionStandardDocument:
57+ return h.handleStandardDocument(ctx, event)
5558 }
5659 return nil
5760 }
@@ -267,3 +270,55 @@ func (h *StreamDBHandler) resolveFeedURL(ctx context.Context, articleURL string)
267270 }
268271 return article.FeedURL
269272 }
273+
274+func (h *StreamDBHandler) handleStandardDocument(ctx context.Context, event *Event) error {
275+ switch event.Type {
276+ case actionCreate, actionUpdate:
277+ var doc StandardDocumentRecord
278+ if err := json.Unmarshal(event.Value, &doc); err != nil {
279+ return err
280+ }
281+ if doc.Title == "" || doc.Site == "" {
282+ return nil
283+ }
284+
285+ publicationURI := doc.Site
286+ if !IsATProtoFeedURL(publicationURI) {
287+ return nil
288+ }
289+
290+ parsed, ok := ParseRecordURI(publicationURI)
291+ if !ok || parsed.Collection != CollectionStandardPublication {
292+ return nil
293+ }
294+
295+ published := parseRFC3339(doc.PublishedAt)
296+ updated := parseRFC3339(doc.UpdatedAt)
297+
298+ _ = h.articles.UpsertFeed(ctx, &db.Feed{
299+ FeedURL: publicationURI,
300+ FeedType: sql.NullString{String: "atproto", Valid: true},
301+ })
302+
303+ var articleURL string
304+ if f, err := h.articles.GetFeed(ctx, publicationURI); err == nil && f.SiteURL.Valid {
305+ articleURL = f.SiteURL.String + doc.Path
306+ }
307+
308+ articles := []feed.Article{{
309+ FeedURL: publicationURI,
310+ GUID: event.URI,
311+ Title: doc.Title,
312+ URL: articleURL,
313+ Content: doc.TextContent,
314+ Summary: doc.Description,
315+ Published: published,
316+ Updated: updated,
317+ }}
318+ return h.articles.BatchUpsertArticles(ctx, articles)
319+
320+ case actionDelete:
321+ return h.articles.DeleteArticleByGUID(ctx, event.URI)
322+ }
323+ return nil
324+}
@@ -9,6 +9,7 @@ import (
9 "time"9 "time"
10 10
11 "pkg.rbrt.fr/glean/internal/db"11 "pkg.rbrt.fr/glean/internal/db"
12+ "pkg.rbrt.fr/glean/internal/feed"
12 )13 )
13 14
14 const (15 const (
@@ -52,6 +53,8 @@ func (h *StreamDBHandler) Handle(ctx context.Context, event *Event) error {
52 return h.handleMarginNote(ctx, event)53 return h.handleMarginNote(ctx, event)
53 case CollectionBskyFollow, CollectionTangledFollow:54 case CollectionBskyFollow, CollectionTangledFollow:
54 return h.handleFollow(ctx, event)55 return h.handleFollow(ctx, event)
56+ case CollectionStandardDocument:
57+ return h.handleStandardDocument(ctx, event)
55 }58 }
56 return nil59 return nil
57 }60 }
@@ -267,3 +270,55 @@ func (h *StreamDBHandler) resolveFeedURL(ctx context.Context, articleURL string)
267 }270 }
268 return article.FeedURL271 return article.FeedURL
269 }272 }
273+
274+func (h *StreamDBHandler) handleStandardDocument(ctx context.Context, event *Event) error {
275+ switch event.Type {
276+ case actionCreate, actionUpdate:
277+ var doc StandardDocumentRecord
278+ if err := json.Unmarshal(event.Value, &doc); err != nil {
279+ return err
280+ }
281+ if doc.Title == "" || doc.Site == "" {
282+ return nil
283+ }
284+
285+ publicationURI := doc.Site
286+ if !IsATProtoFeedURL(publicationURI) {
287+ return nil
288+ }
289+
290+ parsed, ok := ParseRecordURI(publicationURI)
291+ if !ok || parsed.Collection != CollectionStandardPublication {
292+ return nil
293+ }
294+
295+ published := parseRFC3339(doc.PublishedAt)
296+ updated := parseRFC3339(doc.UpdatedAt)
297+
298+ _ = h.articles.UpsertFeed(ctx, &db.Feed{
299+ FeedURL: publicationURI,
300+ FeedType: sql.NullString{String: "atproto", Valid: true},
301+ })
302+
303+ var articleURL string
304+ if f, err := h.articles.GetFeed(ctx, publicationURI); err == nil && f.SiteURL.Valid {
305+ articleURL = f.SiteURL.String + doc.Path
306+ }
307+
308+ articles := []feed.Article{{
309+ FeedURL: publicationURI,
310+ GUID: event.URI,
311+ Title: doc.Title,
312+ URL: articleURL,
313+ Content: doc.TextContent,
314+ Summary: doc.Description,
315+ Published: published,
316+ Updated: updated,
317+ }}
318+ return h.articles.BatchUpsertArticles(ctx, articles)
319+
320+ case actionDelete:
321+ return h.articles.DeleteArticleByGUID(ctx, event.URI)
322+ }
323+ return nil
324+}
modified internal/db/article.go +5 -0
@@ -398,6 +398,11 @@ func (s *ArticleStore) GetArticleByURL(ctx context.Context, url string) (*Articl
398398 return a, nil
399399 }
400400
401+func (s *ArticleStore) DeleteArticleByGUID(ctx context.Context, guid string) error {
402+ _, err := s.db.ExecContext(ctx, `DELETE FROM articles.articles WHERE guid = ?`, guid)
403+ return err
404+}
405+
401406 func (s *ArticleStore) CountNewArticles(ctx context.Context, userDID string, since time.Time) (int, error) {
402407 var count int
403408 err := s.db.QueryRowContext(ctx, `
@@ -398,6 +398,11 @@ func (s *ArticleStore) GetArticleByURL(ctx context.Context, url string) (*Articl
398 return a, nil398 return a, nil
399 }399 }
400 400
401+func (s *ArticleStore) DeleteArticleByGUID(ctx context.Context, guid string) error {
402+ _, err := s.db.ExecContext(ctx, `DELETE FROM articles.articles WHERE guid = ?`, guid)
403+ return err
404+}
405+
401 func (s *ArticleStore) CountNewArticles(ctx context.Context, userDID string, since time.Time) (int, error) {406 func (s *ArticleStore) CountNewArticles(ctx context.Context, userDID string, since time.Time) (int, error) {
402 var count int407 var count int
403 err := s.db.QueryRowContext(ctx, `408 err := s.db.QueryRowContext(ctx, `
modified internal/db/db.go +1 -1
@@ -262,7 +262,7 @@ var articlesSchema = []string{
262262 title TEXT,
263263 site_url TEXT,
264264 description TEXT,
265- feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),
265+ feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json', 'atproto')),
266266 last_fetched_at DATETIME,
267267 last_error TEXT,
268268 subscriber_count INTEGER NOT NULL DEFAULT 0,
@@ -262,7 +262,7 @@ var articlesSchema = []string{
262 title TEXT,262 title TEXT,
263 site_url TEXT,263 site_url TEXT,
264 description TEXT,264 description TEXT,
265- feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),265+ feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json', 'atproto')),
266 last_fetched_at DATETIME,266 last_fetched_at DATETIME,
267 last_error TEXT,267 last_error TEXT,
268 subscriber_count INTEGER NOT NULL DEFAULT 0,268 subscriber_count INTEGER NOT NULL DEFAULT 0,
modified internal/feed/fetcher.go +15 -4
@@ -5,6 +5,7 @@ import (
55 "fmt"
66 "log/slog"
77 "net/http"
8+ "strings"
89 "sync"
910 "time"
1011
@@ -18,11 +19,16 @@ const (
1819 baseRetryDelay = 1 * time.Second
1920 )
2021
22+type ATProtoFetcher interface {
23+ FetchFeed(ctx context.Context, feedURL string) (*ParseResult, error)
24+}
25+
2126 type Fetcher struct {
22- httpClient *http.Client
27+ httpClient *http.Client
28+ atprotoFetcher ATProtoFetcher
2329 }
2430
25-func NewFetcher() *Fetcher {
31+func NewFetcher(atprotoFetcher ATProtoFetcher) *Fetcher {
2632 return &Fetcher{
2733 httpClient: &http.Client{
2834 Timeout: 10 * time.Second,
@@ -34,10 +40,15 @@ func NewFetcher() *Fetcher {
3440 return nil
3541 },
3642 },
43+ atprotoFetcher: atprotoFetcher,
3744 }
3845 }
3946
4047 func (f *Fetcher) Fetch(ctx context.Context, feedURL string) (*ParseResult, error) {
48+ if strings.HasPrefix(feedURL, "at://") {
49+ return f.atprotoFetcher.FetchFeed(ctx, feedURL)
50+ }
51+
4152 var lastResp *http.Response
4253 var lastErr error
4354
@@ -122,9 +133,9 @@ type Scheduler struct {
122133 inFlight sync.Map
123134 }
124135
125-func NewScheduler(store FeedStore, logger *slog.Logger, tickInterval, staleInterval time.Duration) *Scheduler {
136+func NewScheduler(store FeedStore, atprotoFetcher ATProtoFetcher, logger *slog.Logger, tickInterval, staleInterval time.Duration) *Scheduler {
126137 return &Scheduler{
127- fetcher: NewFetcher(),
138+ fetcher: NewFetcher(atprotoFetcher),
128139 store: store,
129140 logger: logger,
130141 tickInterval: tickInterval,
@@ -5,6 +5,7 @@ import (
5 "fmt"5 "fmt"
6 "log/slog"6 "log/slog"
7 "net/http"7 "net/http"
8+ "strings"
8 "sync"9 "sync"
9 "time"10 "time"
10 11
@@ -18,11 +19,16 @@ const (
18 baseRetryDelay = 1 * time.Second19 baseRetryDelay = 1 * time.Second
19 )20 )
20 21
22+type ATProtoFetcher interface {
23+ FetchFeed(ctx context.Context, feedURL string) (*ParseResult, error)
24+}
25+
21 type Fetcher struct {26 type Fetcher struct {
22- httpClient *http.Client27+ httpClient *http.Client
28+ atprotoFetcher ATProtoFetcher
23 }29 }
24 30
25-func NewFetcher() *Fetcher {31+func NewFetcher(atprotoFetcher ATProtoFetcher) *Fetcher {
26 return &Fetcher{32 return &Fetcher{
27 httpClient: &http.Client{33 httpClient: &http.Client{
28 Timeout: 10 * time.Second,34 Timeout: 10 * time.Second,
@@ -34,10 +40,15 @@ func NewFetcher() *Fetcher {
34 return nil40 return nil
35 },41 },
36 },42 },
43+ atprotoFetcher: atprotoFetcher,
37 }44 }
38 }45 }
39 46
40 func (f *Fetcher) Fetch(ctx context.Context, feedURL string) (*ParseResult, error) {47 func (f *Fetcher) Fetch(ctx context.Context, feedURL string) (*ParseResult, error) {
48+ if strings.HasPrefix(feedURL, "at://") {
49+ return f.atprotoFetcher.FetchFeed(ctx, feedURL)
50+ }
51+
41 var lastResp *http.Response52 var lastResp *http.Response
42 var lastErr error53 var lastErr error
43 54
@@ -122,9 +133,9 @@ type Scheduler struct {
122 inFlight sync.Map133 inFlight sync.Map
123 }134 }
124 135
125-func NewScheduler(store FeedStore, logger *slog.Logger, tickInterval, staleInterval time.Duration) *Scheduler {136+func NewScheduler(store FeedStore, atprotoFetcher ATProtoFetcher, logger *slog.Logger, tickInterval, staleInterval time.Duration) *Scheduler {
126 return &Scheduler{137 return &Scheduler{
127- fetcher: NewFetcher(),138+ fetcher: NewFetcher(atprotoFetcher),
128 store: store,139 store: store,
129 logger: logger,140 logger: logger,
130 tickInterval: tickInterval,141 tickInterval: tickInterval,
modified internal/server/feeds_handler.go +10 -1
@@ -6,6 +6,7 @@ import (
66 "errors"
77 "fmt"
88 "net/http"
9+ "net/url"
910 "time"
1011
1112 "golang.org/x/sync/errgroup"
@@ -123,8 +124,16 @@ func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
123124 return
124125 }
125126
127+ if !atproto.IsATProtoFeedURL(feedURL) {
128+ u, err := url.Parse(feedURL)
129+ if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
130+ http.Error(w, "invalid feed URL", http.StatusBadRequest)
131+ return
132+ }
133+ }
134+
126135 result, err := s.fetcher.Fetch(r.Context(), feedURL)
127- if err != nil {
136+ if err != nil && !atproto.IsATProtoFeedURL(feedURL) {
128137 result, feedURL, err = s.discoverFeed(r.Context(), feedURL)
129138 }
130139 if err != nil {
@@ -6,6 +6,7 @@ import (
6 "errors"6 "errors"
7 "fmt"7 "fmt"
8 "net/http"8 "net/http"
9+ "net/url"
9 "time"10 "time"
10 11
11 "golang.org/x/sync/errgroup"12 "golang.org/x/sync/errgroup"
@@ -123,8 +124,16 @@ func (s *Server) handleAddFeed(w http.ResponseWriter, r *http.Request) {
123 return124 return
124 }125 }
125 126
127+ if !atproto.IsATProtoFeedURL(feedURL) {
128+ u, err := url.Parse(feedURL)
129+ if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
130+ http.Error(w, "invalid feed URL", http.StatusBadRequest)
131+ return
132+ }
133+ }
134+
126 result, err := s.fetcher.Fetch(r.Context(), feedURL)135 result, err := s.fetcher.Fetch(r.Context(), feedURL)
127- if err != nil {136+ if err != nil && !atproto.IsATProtoFeedURL(feedURL) {
128 result, feedURL, err = s.discoverFeed(r.Context(), feedURL)137 result, feedURL, err = s.discoverFeed(r.Context(), feedURL)
129 }138 }
130 if err != nil {139 if err != nil {
modified internal/server/server.go +3 -3
@@ -73,7 +73,7 @@ type Server struct {
7373 sessionKey []byte
7474 }
7575
76-func New(dbs *db.Store, clientID, callbackURL, addr string, scheduler *feed.Scheduler, engine *cluster.Engine, logger *slog.Logger, sessionKey []byte) *Server {
76+func New(dbs *db.Store, clientID, callbackURL, addr string, scheduler *feed.Scheduler, fetcher *feed.Fetcher, engine *cluster.Engine, logger *slog.Logger, sessionKey []byte) *Server {
7777 oauthStore := db.NewOAuthStore(dbs)
7878
7979 var config oauth.ClientConfig
@@ -91,11 +91,11 @@ func New(dbs *db.Store, clientID, callbackURL, addr string, scheduler *feed.Sche
9191
9292 s := &Server{
9393 dbs: dbs,
94- router: chi.NewRouter(),
94+ router: chi.NewMux(),
9595 logger: logger,
9696 oauth: oauthClient,
9797 oauthStore: oauthStore,
98- fetcher: feed.NewFetcher(),
98+ fetcher: fetcher,
9999 scheduler: scheduler,
100100 engine: engine,
101101 scraper: scraper.New(logger),
@@ -73,7 +73,7 @@ type Server struct {
73 sessionKey []byte73 sessionKey []byte
74 }74 }
75 75
76-func New(dbs *db.Store, clientID, callbackURL, addr string, scheduler *feed.Scheduler, engine *cluster.Engine, logger *slog.Logger, sessionKey []byte) *Server {76+func New(dbs *db.Store, clientID, callbackURL, addr string, scheduler *feed.Scheduler, fetcher *feed.Fetcher, engine *cluster.Engine, logger *slog.Logger, sessionKey []byte) *Server {
77 oauthStore := db.NewOAuthStore(dbs)77 oauthStore := db.NewOAuthStore(dbs)
78 78
79 var config oauth.ClientConfig79 var config oauth.ClientConfig
@@ -91,11 +91,11 @@ func New(dbs *db.Store, clientID, callbackURL, addr string, scheduler *feed.Sche
91 91
92 s := &Server{92 s := &Server{
93 dbs: dbs,93 dbs: dbs,
94- router: chi.NewRouter(),94+ router: chi.NewMux(),
95 logger: logger,95 logger: logger,
96 oauth: oauthClient,96 oauth: oauthClient,
97 oauthStore: oauthStore,97 oauthStore: oauthStore,
98- fetcher: feed.NewFetcher(),98+ fetcher: fetcher,
99 scheduler: scheduler,99 scheduler: scheduler,
100 engine: engine,100 engine: engine,
101 scraper: scraper.New(logger),101 scraper: scraper.New(logger),
modified internal/tmpl/feeds.html +2 -2
@@ -16,7 +16,7 @@
1616 function gleanRefreshStart(){document.getElementById('refresh-indicator').style.display='inline';document.getElementById('refresh-btn').disabled=true}
1717 function gleanRefreshPoll(){setTimeout(function(){htmx.ajax('GET','/feeds/list',{target:'#feed-list',swap:'innerHTML'});document.getElementById('refresh-indicator').style.display='none';document.getElementById('refresh-btn').disabled=false},5000)}
1818 </script>
19-<p class="text-sm text-spot-secondary mb-6">Manage your RSS and Atom subscriptions.</p>
19+<p class="text-sm text-spot-secondary mb-6">Manage your RSS, Atom, and AT Protocol subscriptions.</p>
2020
2121 {{template "dead-feeds.html" (dict "DeadFeeds" .DeadFeeds "CSRFToken" .CSRFToken)}}
2222
@@ -67,7 +67,7 @@ function gleanRefreshPoll(){setTimeout(function(){htmx.ajax('GET','/feeds/list',
6767 hx-on::after-request="if(event.detail.successful)this.reset();else document.getElementById('add-feed-error').textContent=event.detail.xhr.responseText"
6868 class="space-y-3">
6969 {{csrfInput .CSRFToken}}
70- <input type="url" name="feed_url" placeholder="https://example.com/feed.xml"
70+ <input type="text" name="feed_url" placeholder="https://example.com/feed.xml or at://did:plc:.../site.standard.publication/..."
7171 class="w-full bg-spot-hover text-spot-text rounded-pill px-5 py-2 text-sm focus:outline-none focus:ring-2 focus:ring-spot-green placeholder:text-spot-placeholder"
7272 required>
7373 <input type="text" name="category" placeholder="Category (optional)"
@@ -16,7 +16,7 @@
16 function gleanRefreshStart(){document.getElementById('refresh-indicator').style.display='inline';document.getElementById('refresh-btn').disabled=true}16 function gleanRefreshStart(){document.getElementById('refresh-indicator').style.display='inline';document.getElementById('refresh-btn').disabled=true}
17 function gleanRefreshPoll(){setTimeout(function(){htmx.ajax('GET','/feeds/list',{target:'#feed-list',swap:'innerHTML'});document.getElementById('refresh-indicator').style.display='none';document.getElementById('refresh-btn').disabled=false},5000)}17 function gleanRefreshPoll(){setTimeout(function(){htmx.ajax('GET','/feeds/list',{target:'#feed-list',swap:'innerHTML'});document.getElementById('refresh-indicator').style.display='none';document.getElementById('refresh-btn').disabled=false},5000)}
18 </script>18 </script>
19-<p class="text-sm text-spot-secondary mb-6">Manage your RSS and Atom subscriptions.</p>19+<p class="text-sm text-spot-secondary mb-6">Manage your RSS, Atom, and AT Protocol subscriptions.</p>
20 20
21 {{template "dead-feeds.html" (dict "DeadFeeds" .DeadFeeds "CSRFToken" .CSRFToken)}}21 {{template "dead-feeds.html" (dict "DeadFeeds" .DeadFeeds "CSRFToken" .CSRFToken)}}
22 22
@@ -67,7 +67,7 @@ function gleanRefreshPoll(){setTimeout(function(){htmx.ajax('GET','/feeds/list',
67 hx-on::after-request="if(event.detail.successful)this.reset();else document.getElementById('add-feed-error').textContent=event.detail.xhr.responseText"67 hx-on::after-request="if(event.detail.successful)this.reset();else document.getElementById('add-feed-error').textContent=event.detail.xhr.responseText"
68 class="space-y-3">68 class="space-y-3">
69 {{csrfInput .CSRFToken}}69 {{csrfInput .CSRFToken}}
70- <input type="url" name="feed_url" placeholder="https://example.com/feed.xml"70+ <input type="text" name="feed_url" placeholder="https://example.com/feed.xml or at://did:plc:.../site.standard.publication/..."
71 class="w-full bg-spot-hover text-spot-text rounded-pill px-5 py-2 text-sm focus:outline-none focus:ring-2 focus:ring-spot-green placeholder:text-spot-placeholder"71 class="w-full bg-spot-hover text-spot-text rounded-pill px-5 py-2 text-sm focus:outline-none focus:ring-2 focus:ring-spot-green placeholder:text-spot-placeholder"
72 required>72 required>
73 <input type="text" name="category" placeholder="Category (optional)"73 <input type="text" name="category" placeholder="Category (optional)"
added lexicons/site/standard/document.json +125 -0
new file mode 100644
@@ -0,0 +1,125 @@
1+{
2+ "defs": {
3+ "contributor": {
4+ "properties": {
5+ "did": {
6+ "format": "did",
7+ "type": "string"
8+ },
9+ "displayName": {
10+ "maxGraphemes": 100,
11+ "maxLength": 1000,
12+ "type": "string"
13+ },
14+ "role": {
15+ "maxGraphemes": 100,
16+ "maxLength": 1000,
17+ "type": "string"
18+ }
19+ },
20+ "required": [
21+ "did"
22+ ],
23+ "type": "object"
24+ },
25+ "main": {
26+ "description": "A document record representing a published article, blog post, or other content. Documents can belong to a publication or exist independently.",
27+ "key": "tid",
28+ "record": {
29+ "properties": {
30+ "bskyPostRef": {
31+ "description": "Strong reference to a Bluesky post. Useful to keep track of comments off-platform.",
32+ "ref": "com.atproto.repo.strongRef",
33+ "type": "ref"
34+ },
35+ "content": {
36+ "closed": false,
37+ "description": "Open union used to define the record's content. Each entry must specify a $type and may be extended with other lexicons to support additional content formats.",
38+ "refs": [],
39+ "type": "union"
40+ },
41+ "contributors": {
42+ "items": {
43+ "ref": "#contributor",
44+ "type": "ref"
45+ },
46+ "type": "array"
47+ },
48+ "coverImage": {
49+ "accept": [
50+ "image/*"
51+ ],
52+ "description": "Image to used for thumbnail or cover image. Less than 1MB is size.",
53+ "maxSize": 1000000,
54+ "type": "blob"
55+ },
56+ "description": {
57+ "description": "A brief description or excerpt from the document.",
58+ "maxGraphemes": 3000,
59+ "maxLength": 30000,
60+ "type": "string"
61+ },
62+ "labels": {
63+ "description": "Self-label values for this post. Effectively content warnings.",
64+ "refs": [
65+ "com.atproto.label.defs#selfLabels"
66+ ],
67+ "type": "union"
68+ },
69+ "links": {
70+ "description": "Array of values describing relationships between this document and external resources",
71+ "refs": [],
72+ "type": "union"
73+ },
74+ "path": {
75+ "description": "Combine with site or publication url to construct a canonical URL to the document. Prepend with a leading slash.",
76+ "type": "string"
77+ },
78+ "publishedAt": {
79+ "description": "Timestamp of the documents publish time.",
80+ "format": "datetime",
81+ "type": "string"
82+ },
83+ "site": {
84+ "description": "Points to a publication record (at://) or a publication url (https://) for loose documents. Avoid trailing slashes.",
85+ "format": "uri",
86+ "type": "string"
87+ },
88+ "tags": {
89+ "description": "Array of strings used to tag or categorize the document. Avoid prepending tags with hashtags.",
90+ "items": {
91+ "maxGraphemes": 128,
92+ "maxLength": 1280,
93+ "type": "string"
94+ },
95+ "type": "array"
96+ },
97+ "textContent": {
98+ "description": "Plaintext representation of the documents contents. Should not contain markdown or other formatting.",
99+ "type": "string"
100+ },
101+ "title": {
102+ "description": "Title of the document.",
103+ "maxGraphemes": 500,
104+ "maxLength": 5000,
105+ "type": "string"
106+ },
107+ "updatedAt": {
108+ "description": "Timestamp of the documents last edit.",
109+ "format": "datetime",
110+ "type": "string"
111+ }
112+ },
113+ "required": [
114+ "site",
115+ "title",
116+ "publishedAt"
117+ ],
118+ "type": "object"
119+ },
120+ "type": "record"
121+ }
122+ },
123+ "id": "site.standard.document",
124+ "lexicon": 1
125+}
new file mode 100644
@@ -0,0 +1,125 @@
1+{
2+ "defs": {
3+ "contributor": {
4+ "properties": {
5+ "did": {
6+ "format": "did",
7+ "type": "string"
8+ },
9+ "displayName": {
10+ "maxGraphemes": 100,
11+ "maxLength": 1000,
12+ "type": "string"
13+ },
14+ "role": {
15+ "maxGraphemes": 100,
16+ "maxLength": 1000,
17+ "type": "string"
18+ }
19+ },
20+ "required": [
21+ "did"
22+ ],
23+ "type": "object"
24+ },
25+ "main": {
26+ "description": "A document record representing a published article, blog post, or other content. Documents can belong to a publication or exist independently.",
27+ "key": "tid",
28+ "record": {
29+ "properties": {
30+ "bskyPostRef": {
31+ "description": "Strong reference to a Bluesky post. Useful to keep track of comments off-platform.",
32+ "ref": "com.atproto.repo.strongRef",
33+ "type": "ref"
34+ },
35+ "content": {
36+ "closed": false,
37+ "description": "Open union used to define the record's content. Each entry must specify a $type and may be extended with other lexicons to support additional content formats.",
38+ "refs": [],
39+ "type": "union"
40+ },
41+ "contributors": {
42+ "items": {
43+ "ref": "#contributor",
44+ "type": "ref"
45+ },
46+ "type": "array"
47+ },
48+ "coverImage": {
49+ "accept": [
50+ "image/*"
51+ ],
52+ "description": "Image to used for thumbnail or cover image. Less than 1MB is size.",
53+ "maxSize": 1000000,
54+ "type": "blob"
55+ },
56+ "description": {
57+ "description": "A brief description or excerpt from the document.",
58+ "maxGraphemes": 3000,
59+ "maxLength": 30000,
60+ "type": "string"
61+ },
62+ "labels": {
63+ "description": "Self-label values for this post. Effectively content warnings.",
64+ "refs": [
65+ "com.atproto.label.defs#selfLabels"
66+ ],
67+ "type": "union"
68+ },
69+ "links": {
70+ "description": "Array of values describing relationships between this document and external resources",
71+ "refs": [],
72+ "type": "union"
73+ },
74+ "path": {
75+ "description": "Combine with site or publication url to construct a canonical URL to the document. Prepend with a leading slash.",
76+ "type": "string"
77+ },
78+ "publishedAt": {
79+ "description": "Timestamp of the documents publish time.",
80+ "format": "datetime",
81+ "type": "string"
82+ },
83+ "site": {
84+ "description": "Points to a publication record (at://) or a publication url (https://) for loose documents. Avoid trailing slashes.",
85+ "format": "uri",
86+ "type": "string"
87+ },
88+ "tags": {
89+ "description": "Array of strings used to tag or categorize the document. Avoid prepending tags with hashtags.",
90+ "items": {
91+ "maxGraphemes": 128,
92+ "maxLength": 1280,
93+ "type": "string"
94+ },
95+ "type": "array"
96+ },
97+ "textContent": {
98+ "description": "Plaintext representation of the documents contents. Should not contain markdown or other formatting.",
99+ "type": "string"
100+ },
101+ "title": {
102+ "description": "Title of the document.",
103+ "maxGraphemes": 500,
104+ "maxLength": 5000,
105+ "type": "string"
106+ },
107+ "updatedAt": {
108+ "description": "Timestamp of the documents last edit.",
109+ "format": "datetime",
110+ "type": "string"
111+ }
112+ },
113+ "required": [
114+ "site",
115+ "title",
116+ "publishedAt"
117+ ],
118+ "type": "object"
119+ },
120+ "type": "record"
121+ }
122+ },
123+ "id": "site.standard.document",
124+ "lexicon": 1
125+}
added lexicons/site/standard/publication.json +72 -0
new file mode 100644
@@ -0,0 +1,72 @@
1+{
2+ "defs": {
3+ "main": {
4+ "description": "A publication record representing a blog, website, or content platform. Publications serve as containers for documents and define the overall branding and settings.",
5+ "key": "tid",
6+ "record": {
7+ "properties": {
8+ "basicTheme": {
9+ "description": "Simplified publication theme for tools and apps to utilize when displaying content.",
10+ "ref": "site.standard.theme.basic",
11+ "type": "ref"
12+ },
13+ "description": {
14+ "description": "Brief description of the publication.",
15+ "maxGraphemes": 3000,
16+ "maxLength": 30000,
17+ "type": "string"
18+ },
19+ "icon": {
20+ "accept": [
21+ "image/*"
22+ ],
23+ "description": "Square image to identify the publication. Should be at least 256x256.",
24+ "maxSize": 1000000,
25+ "type": "blob"
26+ },
27+ "labels": {
28+ "description": "Self-label values for this publication. Effectively content warnings.",
29+ "refs": [
30+ "com.atproto.label.defs#selfLabels"
31+ ],
32+ "type": "union"
33+ },
34+ "name": {
35+ "description": "Name of the publication.",
36+ "maxGraphemes": 500,
37+ "maxLength": 5000,
38+ "type": "string"
39+ },
40+ "preferences": {
41+ "description": "Object containing platform specific preferences (with a few shared properties).",
42+ "ref": "#preferences",
43+ "type": "ref"
44+ },
45+ "url": {
46+ "description": "Base publication url (ex: https://standard.site). The canonical document URL is formed by combining this value with the document path.",
47+ "format": "uri",
48+ "type": "string"
49+ }
50+ },
51+ "required": [
52+ "url",
53+ "name"
54+ ],
55+ "type": "object"
56+ },
57+ "type": "record"
58+ },
59+ "preferences": {
60+ "properties": {
61+ "showInDiscover": {
62+ "default": true,
63+ "description": "Boolean which decides whether the publication should appear in discovery feeds.",
64+ "type": "boolean"
65+ }
66+ },
67+ "type": "object"
68+ }
69+ },
70+ "id": "site.standard.publication",
71+ "lexicon": 1
72+}
new file mode 100644
@@ -0,0 +1,72 @@
1+{
2+ "defs": {
3+ "main": {
4+ "description": "A publication record representing a blog, website, or content platform. Publications serve as containers for documents and define the overall branding and settings.",
5+ "key": "tid",
6+ "record": {
7+ "properties": {
8+ "basicTheme": {
9+ "description": "Simplified publication theme for tools and apps to utilize when displaying content.",
10+ "ref": "site.standard.theme.basic",
11+ "type": "ref"
12+ },
13+ "description": {
14+ "description": "Brief description of the publication.",
15+ "maxGraphemes": 3000,
16+ "maxLength": 30000,
17+ "type": "string"
18+ },
19+ "icon": {
20+ "accept": [
21+ "image/*"
22+ ],
23+ "description": "Square image to identify the publication. Should be at least 256x256.",
24+ "maxSize": 1000000,
25+ "type": "blob"
26+ },
27+ "labels": {
28+ "description": "Self-label values for this publication. Effectively content warnings.",
29+ "refs": [
30+ "com.atproto.label.defs#selfLabels"
31+ ],
32+ "type": "union"
33+ },
34+ "name": {
35+ "description": "Name of the publication.",
36+ "maxGraphemes": 500,
37+ "maxLength": 5000,
38+ "type": "string"
39+ },
40+ "preferences": {
41+ "description": "Object containing platform specific preferences (with a few shared properties).",
42+ "ref": "#preferences",
43+ "type": "ref"
44+ },
45+ "url": {
46+ "description": "Base publication url (ex: https://standard.site). The canonical document URL is formed by combining this value with the document path.",
47+ "format": "uri",
48+ "type": "string"
49+ }
50+ },
51+ "required": [
52+ "url",
53+ "name"
54+ ],
55+ "type": "object"
56+ },
57+ "type": "record"
58+ },
59+ "preferences": {
60+ "properties": {
61+ "showInDiscover": {
62+ "default": true,
63+ "description": "Boolean which decides whether the publication should appear in discovery feeds.",
64+ "type": "boolean"
65+ }
66+ },
67+ "type": "object"
68+ }
69+ },
70+ "id": "site.standard.publication",
71+ "lexicon": 1
72+}
modified main.go +4 -2
@@ -54,7 +54,8 @@ func main() {
5454 callbackURL := envOr("GLEAN_OAUTH_REDIRECT_URL", "")
5555
5656 storeAdapter := db.NewFeedAdapter(dbs.Articles)
57- scheduler := feed.NewScheduler(storeAdapter, logger, *fetchInterval, 30*time.Minute)
57+ siteFetcher := atproto.NewStandardSiteFetcher(logger)
58+ scheduler := feed.NewScheduler(storeAdapter, siteFetcher, logger, *fetchInterval, 30*time.Minute)
5859
5960 var embedder cluster.Embedder
6061 if embedURL := envOr("GLEAN_EMBED_BASE_URL", ""); embedURL != "" {
@@ -75,7 +76,8 @@ func main() {
7576
7677 engine := cluster.NewEngine(dbs.SQLDB(), embedder, logger)
7778
78- srv := server.New(dbs, clientID, callbackURL, *addr, scheduler, engine, logger, []byte(sessionKey))
79+ fetcher := feed.NewFetcher(siteFetcher)
80+ srv := server.New(dbs, clientID, callbackURL, *addr, scheduler, fetcher, engine, logger, []byte(sessionKey))
7981
8082 cron := cluster.NewCron(engine, *clusterInterval, logger)
8183
@@ -54,7 +54,8 @@ func main() {
54 callbackURL := envOr("GLEAN_OAUTH_REDIRECT_URL", "")54 callbackURL := envOr("GLEAN_OAUTH_REDIRECT_URL", "")
55 55
56 storeAdapter := db.NewFeedAdapter(dbs.Articles)56 storeAdapter := db.NewFeedAdapter(dbs.Articles)
57- scheduler := feed.NewScheduler(storeAdapter, logger, *fetchInterval, 30*time.Minute)57+ siteFetcher := atproto.NewStandardSiteFetcher(logger)
58+ scheduler := feed.NewScheduler(storeAdapter, siteFetcher, logger, *fetchInterval, 30*time.Minute)
58 59
59 var embedder cluster.Embedder60 var embedder cluster.Embedder
60 if embedURL := envOr("GLEAN_EMBED_BASE_URL", ""); embedURL != "" {61 if embedURL := envOr("GLEAN_EMBED_BASE_URL", ""); embedURL != "" {
@@ -75,7 +76,8 @@ func main() {
75 76
76 engine := cluster.NewEngine(dbs.SQLDB(), embedder, logger)77 engine := cluster.NewEngine(dbs.SQLDB(), embedder, logger)
77 78
78- srv := server.New(dbs, clientID, callbackURL, *addr, scheduler, engine, logger, []byte(sessionKey))79+ fetcher := feed.NewFetcher(siteFetcher)
80+ srv := server.New(dbs, clientID, callbackURL, *addr, scheduler, fetcher, engine, logger, []byte(sessionKey))
79 81
80 cron := cluster.NewCron(engine, *clusterInterval, logger)82 cron := cluster.NewCron(engine, *clusterInterval, logger)
81 83