rickub
nandi/gleanpublic⑂ Fork 0
⑂ a285def
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Improve profile fetchingUnverified

Julien Robert committed 2026-04-22T12:30:44+02:00 Browse files
a285def parent: 69cac9a
modified internal/atproto/client.go +16 -0
@@ -99,3 +99,19 @@ func (c *Client) ListRecords(ctx context.Context, did, collection string, limit
9999 }
100100 return records, result.Cursor, nil
101101 }
102+
103+func (c *Client) GetProfile(ctx context.Context, did string) (displayName, avatarURL string, err error) {
104+ nsid, err := syntax.ParseNSID("app.bsky.actor.getProfile")
105+ if err != nil {
106+ return "", "", fmt.Errorf("parsing NSID: %w", err)
107+ }
108+
109+ var profile struct {
110+ DisplayName string `json:"displayName"`
111+ Avatar string `json:"avatar"`
112+ }
113+ if err := c.api.Get(ctx, nsid, map[string]any{"actor": did}, &profile); err != nil {
114+ return "", "", err
115+ }
116+ return profile.DisplayName, profile.Avatar, nil
117+}
@@ -99,3 +99,19 @@ func (c *Client) ListRecords(ctx context.Context, did, collection string, limit
99 }99 }
100 return records, result.Cursor, nil100 return records, result.Cursor, nil
101 }101 }
102+
103+func (c *Client) GetProfile(ctx context.Context, did string) (displayName, avatarURL string, err error) {
104+ nsid, err := syntax.ParseNSID("app.bsky.actor.getProfile")
105+ if err != nil {
106+ return "", "", fmt.Errorf("parsing NSID: %w", err)
107+ }
108+
109+ var profile struct {
110+ DisplayName string `json:"displayName"`
111+ Avatar string `json:"avatar"`
112+ }
113+ if err := c.api.Get(ctx, nsid, map[string]any{"actor": did}, &profile); err != nil {
114+ return "", "", err
115+ }
116+ return profile.DisplayName, profile.Avatar, nil
117+}
added internal/atproto/profile.go +45 -0
new file mode 100644
@@ -0,0 +1,45 @@
1+package atproto
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "fmt"
7+ "net/http"
8+ "net/url"
9+ "time"
10+)
11+
12+var profileClient = &http.Client{Timeout: 10 * time.Second}
13+
14+// FetchProfile is an unauthenticated profile fetcher. It relies on bluesky api.
15+func FetchProfile(ctx context.Context, identifier string) (handle, displayName, avatarURL string, err error) {
16+ apiURL := fmt.Sprintf("https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=%s", url.QueryEscape(identifier))
17+ return fetchProfileFromURL(ctx, apiURL, identifier)
18+}
19+
20+func fetchProfileFromURL(ctx context.Context, apiURL, identifier string) (handle, displayName, avatarURL string, err error) {
21+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
22+ if err != nil {
23+ return "", "", "", err
24+ }
25+
26+ resp, err := profileClient.Do(req)
27+ if err != nil {
28+ return "", "", "", err
29+ }
30+ defer resp.Body.Close()
31+
32+ if resp.StatusCode != http.StatusOK {
33+ return "", "", "", fmt.Errorf("fetch profile %s: status %d", identifier, resp.StatusCode)
34+ }
35+
36+ var profile struct {
37+ Handle string `json:"handle"`
38+ DisplayName string `json:"displayName"`
39+ Avatar string `json:"avatar"`
40+ }
41+ if err := json.NewDecoder(resp.Body).Decode(&profile); err != nil {
42+ return "", "", "", err
43+ }
44+ return profile.Handle, profile.DisplayName, profile.Avatar, nil
45+}
new file mode 100644
@@ -0,0 +1,45 @@
1+package atproto
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "fmt"
7+ "net/http"
8+ "net/url"
9+ "time"
10+)
11+
12+var profileClient = &http.Client{Timeout: 10 * time.Second}
13+
14+// FetchProfile is an unauthenticated profile fetcher. It relies on bluesky api.
15+func FetchProfile(ctx context.Context, identifier string) (handle, displayName, avatarURL string, err error) {
16+ apiURL := fmt.Sprintf("https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=%s", url.QueryEscape(identifier))
17+ return fetchProfileFromURL(ctx, apiURL, identifier)
18+}
19+
20+func fetchProfileFromURL(ctx context.Context, apiURL, identifier string) (handle, displayName, avatarURL string, err error) {
21+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
22+ if err != nil {
23+ return "", "", "", err
24+ }
25+
26+ resp, err := profileClient.Do(req)
27+ if err != nil {
28+ return "", "", "", err
29+ }
30+ defer resp.Body.Close()
31+
32+ if resp.StatusCode != http.StatusOK {
33+ return "", "", "", fmt.Errorf("fetch profile %s: status %d", identifier, resp.StatusCode)
34+ }
35+
36+ var profile struct {
37+ Handle string `json:"handle"`
38+ DisplayName string `json:"displayName"`
39+ Avatar string `json:"avatar"`
40+ }
41+ if err := json.NewDecoder(resp.Body).Decode(&profile); err != nil {
42+ return "", "", "", err
43+ }
44+ return profile.Handle, profile.DisplayName, profile.Avatar, nil
45+}
added internal/atproto/profile_test.go +66 -0
new file mode 100644
@@ -0,0 +1,66 @@
1+package atproto
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "net/http"
7+ "net/http/httptest"
8+ "net/url"
9+ "testing"
10+
11+ "gotest.tools/v3/assert"
12+)
13+
14+func TestFetchProfile_Success(t *testing.T) {
15+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
16+ assert.Equal(t, r.URL.Query().Get("actor"), "did:plc:test123")
17+ json.NewEncoder(w).Encode(map[string]string{
18+ "handle": "test.bsky.social",
19+ "displayName": "Test User",
20+ "avatar": "https://cdn.bsky.app/img/avatar/test.png",
21+ })
22+ }))
23+ defer srv.Close()
24+
25+ origClient := profileClient
26+ profileClient = srv.Client()
27+ defer func() { profileClient = origClient }()
28+
29+ apiURL := srv.URL + "/xrpc/app.bsky.actor.getProfile?actor=" + url.QueryEscape("did:plc:test123")
30+ handle, dn, avatar, err := fetchProfileFromURL(context.Background(), apiURL, "did:plc:test123")
31+ assert.NilError(t, err)
32+ assert.Equal(t, handle, "test.bsky.social")
33+ assert.Equal(t, dn, "Test User")
34+ assert.Equal(t, avatar, "https://cdn.bsky.app/img/avatar/test.png")
35+}
36+
37+func TestFetchProfile_EmptyProfile(t *testing.T) {
38+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
39+ json.NewEncoder(w).Encode(map[string]string{})
40+ }))
41+ defer srv.Close()
42+
43+ origClient := profileClient
44+ profileClient = srv.Client()
45+ defer func() { profileClient = origClient }()
46+
47+ handle, dn, avatar, err := fetchProfileFromURL(context.Background(), srv.URL+"/xrpc/app.bsky.actor.getProfile", "did:plc:test123")
48+ assert.NilError(t, err)
49+ assert.Equal(t, handle, "")
50+ assert.Equal(t, dn, "")
51+ assert.Equal(t, avatar, "")
52+}
53+
54+func TestFetchProfile_Non200(t *testing.T) {
55+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
56+ w.WriteHeader(http.StatusNotFound)
57+ }))
58+ defer srv.Close()
59+
60+ origClient := profileClient
61+ profileClient = srv.Client()
62+ defer func() { profileClient = origClient }()
63+
64+ _, _, _, err := fetchProfileFromURL(context.Background(), srv.URL+"/xrpc/app.bsky.actor.getProfile", "did:plc:test123")
65+ assert.Assert(t, err != nil)
66+}
new file mode 100644
@@ -0,0 +1,66 @@
1+package atproto
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "net/http"
7+ "net/http/httptest"
8+ "net/url"
9+ "testing"
10+
11+ "gotest.tools/v3/assert"
12+)
13+
14+func TestFetchProfile_Success(t *testing.T) {
15+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
16+ assert.Equal(t, r.URL.Query().Get("actor"), "did:plc:test123")
17+ json.NewEncoder(w).Encode(map[string]string{
18+ "handle": "test.bsky.social",
19+ "displayName": "Test User",
20+ "avatar": "https://cdn.bsky.app/img/avatar/test.png",
21+ })
22+ }))
23+ defer srv.Close()
24+
25+ origClient := profileClient
26+ profileClient = srv.Client()
27+ defer func() { profileClient = origClient }()
28+
29+ apiURL := srv.URL + "/xrpc/app.bsky.actor.getProfile?actor=" + url.QueryEscape("did:plc:test123")
30+ handle, dn, avatar, err := fetchProfileFromURL(context.Background(), apiURL, "did:plc:test123")
31+ assert.NilError(t, err)
32+ assert.Equal(t, handle, "test.bsky.social")
33+ assert.Equal(t, dn, "Test User")
34+ assert.Equal(t, avatar, "https://cdn.bsky.app/img/avatar/test.png")
35+}
36+
37+func TestFetchProfile_EmptyProfile(t *testing.T) {
38+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
39+ json.NewEncoder(w).Encode(map[string]string{})
40+ }))
41+ defer srv.Close()
42+
43+ origClient := profileClient
44+ profileClient = srv.Client()
45+ defer func() { profileClient = origClient }()
46+
47+ handle, dn, avatar, err := fetchProfileFromURL(context.Background(), srv.URL+"/xrpc/app.bsky.actor.getProfile", "did:plc:test123")
48+ assert.NilError(t, err)
49+ assert.Equal(t, handle, "")
50+ assert.Equal(t, dn, "")
51+ assert.Equal(t, avatar, "")
52+}
53+
54+func TestFetchProfile_Non200(t *testing.T) {
55+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
56+ w.WriteHeader(http.StatusNotFound)
57+ }))
58+ defer srv.Close()
59+
60+ origClient := profileClient
61+ profileClient = srv.Client()
62+ defer func() { profileClient = origClient }()
63+
64+ _, _, _, err := fetchProfileFromURL(context.Background(), srv.URL+"/xrpc/app.bsky.actor.getProfile", "did:plc:test123")
65+ assert.Assert(t, err != nil)
66+}
modified internal/atproto/sync.go +8 -3
@@ -266,10 +266,15 @@ func (s *Sync) syncFollows(ctx context.Context, userDID string) error {
266266 FollowedAt: db.NullTime(t),
267267 }
268268
269- var handle string
270- if rec.Subject != userDID {
271- s.db.CreateUser(ctx, rec.Subject, handle, "", "")
269+ // auto onboard followers
270+ var handle, displayName, avatarURL string
271+ if h, dn, avatar, err := FetchProfile(ctx, rec.Subject); err == nil {
272+ handle = h
273+ displayName = dn
274+ avatarURL = avatar
272275 }
276+
277+ s.db.CreateUser(ctx, rec.Subject, handle, displayName, avatarURL)
273278 }
274279
275280 if next == "" || len(records) == 0 {
@@ -266,10 +266,15 @@ func (s *Sync) syncFollows(ctx context.Context, userDID string) error {
266 FollowedAt: db.NullTime(t),266 FollowedAt: db.NullTime(t),
267 }267 }
268 268
269- var handle string269+ // auto onboard followers
270- if rec.Subject != userDID {270+ var handle, displayName, avatarURL string
271- s.db.CreateUser(ctx, rec.Subject, handle, "", "")271+ if h, dn, avatar, err := FetchProfile(ctx, rec.Subject); err == nil {
272+ handle = h
273+ displayName = dn
274+ avatarURL = avatar
272 }275 }
276+
277+ s.db.CreateUser(ctx, rec.Subject, handle, displayName, avatarURL)
273 }278 }
274 279
275 if next == "" || len(records) == 0 {280 if next == "" || len(records) == 0 {
modified internal/db/user.go +11 -0
@@ -72,6 +72,17 @@ func (db *DB) ListUserDIDs(ctx context.Context) (map[string]bool, error) {
7272 return dids, rows.Err()
7373 }
7474
75+func (db *DB) UpdateUserProfile(ctx context.Context, did, displayName, avatarURL string) error {
76+ _, err := db.ExecContext(ctx, `
77+ UPDATE users SET
78+ display_name = COALESCE(NULLIF(?, ''), display_name),
79+ avatar_url = COALESCE(NULLIF(?, ''), avatar_url),
80+ updated_at = CURRENT_TIMESTAMP
81+ WHERE did = ?
82+ `, displayName, avatarURL, did)
83+ return err
84+}
85+
7586 func (db *DB) ListUsers(ctx context.Context) ([]*User, error) {
7687 rows, err := db.QueryContext(ctx, `
7788 SELECT did, handle, display_name, avatar_url, indexed_at, updated_at
@@ -72,6 +72,17 @@ func (db *DB) ListUserDIDs(ctx context.Context) (map[string]bool, error) {
72 return dids, rows.Err()72 return dids, rows.Err()
73 }73 }
74 74
75+func (db *DB) UpdateUserProfile(ctx context.Context, did, displayName, avatarURL string) error {
76+ _, err := db.ExecContext(ctx, `
77+ UPDATE users SET
78+ display_name = COALESCE(NULLIF(?, ''), display_name),
79+ avatar_url = COALESCE(NULLIF(?, ''), avatar_url),
80+ updated_at = CURRENT_TIMESTAMP
81+ WHERE did = ?
82+ `, displayName, avatarURL, did)
83+ return err
84+}
85+
75 func (db *DB) ListUsers(ctx context.Context) ([]*User, error) {86 func (db *DB) ListUsers(ctx context.Context) ([]*User, error) {
76 rows, err := db.QueryContext(ctx, `87 rows, err := db.QueryContext(ctx, `
77 SELECT did, handle, display_name, avatar_url, indexed_at, updated_at88 SELECT did, handle, display_name, avatar_url, indexed_at, updated_at
added internal/db/user_test.go +41 -0
new file mode 100644
@@ -0,0 +1,41 @@
1+package db
2+
3+import (
4+ "context"
5+ "testing"
6+
7+ "gotest.tools/v3/assert"
8+)
9+
10+func TestUpdateUserProfile_SetsFields(t *testing.T) {
11+ ctx := context.Background()
12+ db := setupTestDB(t)
13+
14+ _, err := db.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:profile", "tester")
15+ assert.NilError(t, err)
16+
17+ err = db.UpdateUserProfile(ctx, "did:test:profile", "Display Name", "https://cdn.bsky.app/img/avatar.png")
18+ assert.NilError(t, err)
19+
20+ u, err := db.GetUser(ctx, "did:test:profile")
21+ assert.NilError(t, err)
22+ assert.Equal(t, u.DisplayName.String, "Display Name")
23+ assert.Equal(t, u.AvatarURL.String, "https://cdn.bsky.app/img/avatar.png")
24+}
25+
26+func TestUpdateUserProfile_DoesNotOverwriteWithEmpty(t *testing.T) {
27+ ctx := context.Background()
28+ db := setupTestDB(t)
29+
30+ _, err := db.ExecContext(ctx, `INSERT INTO users (did, handle, display_name, avatar_url) VALUES (?, ?, ?, ?)`,
31+ "did:test:profile2", "tester2", "Existing Name", "https://old.avatar/url")
32+ assert.NilError(t, err)
33+
34+ err = db.UpdateUserProfile(ctx, "did:test:profile2", "", "")
35+ assert.NilError(t, err)
36+
37+ u, err := db.GetUser(ctx, "did:test:profile2")
38+ assert.NilError(t, err)
39+ assert.Equal(t, u.DisplayName.String, "Existing Name")
40+ assert.Equal(t, u.AvatarURL.String, "https://old.avatar/url")
41+}
new file mode 100644
@@ -0,0 +1,41 @@
1+package db
2+
3+import (
4+ "context"
5+ "testing"
6+
7+ "gotest.tools/v3/assert"
8+)
9+
10+func TestUpdateUserProfile_SetsFields(t *testing.T) {
11+ ctx := context.Background()
12+ db := setupTestDB(t)
13+
14+ _, err := db.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:profile", "tester")
15+ assert.NilError(t, err)
16+
17+ err = db.UpdateUserProfile(ctx, "did:test:profile", "Display Name", "https://cdn.bsky.app/img/avatar.png")
18+ assert.NilError(t, err)
19+
20+ u, err := db.GetUser(ctx, "did:test:profile")
21+ assert.NilError(t, err)
22+ assert.Equal(t, u.DisplayName.String, "Display Name")
23+ assert.Equal(t, u.AvatarURL.String, "https://cdn.bsky.app/img/avatar.png")
24+}
25+
26+func TestUpdateUserProfile_DoesNotOverwriteWithEmpty(t *testing.T) {
27+ ctx := context.Background()
28+ db := setupTestDB(t)
29+
30+ _, err := db.ExecContext(ctx, `INSERT INTO users (did, handle, display_name, avatar_url) VALUES (?, ?, ?, ?)`,
31+ "did:test:profile2", "tester2", "Existing Name", "https://old.avatar/url")
32+ assert.NilError(t, err)
33+
34+ err = db.UpdateUserProfile(ctx, "did:test:profile2", "", "")
35+ assert.NilError(t, err)
36+
37+ u, err := db.GetUser(ctx, "did:test:profile2")
38+ assert.NilError(t, err)
39+ assert.Equal(t, u.DisplayName.String, "Existing Name")
40+ assert.Equal(t, u.AvatarURL.String, "https://old.avatar/url")
41+}
modified internal/feed/discover.go +64 -127
@@ -2,11 +2,10 @@ package feed
22
33 import (
44 "context"
5- "fmt"
65 "io"
76 "net/http"
7+ "net/url"
88 "regexp"
9- "strings"
109 "time"
1110 )
1211
@@ -16,183 +15,121 @@ type DiscoveryResult struct {
1615 }
1716
1817 var (
19- linkRe = regexp.MustCompile(`<link[^>]+>`)
20- hrefRe = regexp.MustCompile(`href="([^"]*)"`)
21- relFeedRe = regexp.MustCompile(`rel="(alternate|feed)"`)
22- typeFeedRe = regexp.MustCompile(`type="([^"]*(?:rss|atom|feed|xml)[^"]*)"`)
23- relIconRe = regexp.MustCompile(`rel="[^"]*icon[^"]*"`)
24- baseHrefRe = regexp.MustCompile(`<base[^>]+href="([^"]*)"`)
25- faviconPaths = []string{"/favicon.ico", "/favicon.png", "/apple-touch-icon.png"}
26-
18+ linkRe = regexp.MustCompile(`<link[^>]+>`)
19+ hrefRe = regexp.MustCompile(`href="([^"]*)"`)
20+ relFeedRe = regexp.MustCompile(`rel="(alternate|feed)"`)
21+ typeFeedRe = regexp.MustCompile(`type="[^"]*(?:rss|atom|feed|xml)[^"]*"`)
22+ relIconRe = regexp.MustCompile(`rel="[^"]*icon[^"]*"`)
23+ baseHrefRe = regexp.MustCompile(`<base[^>]+href="([^"]*)"`)
24+
25+ faviconPaths = []string{"/favicon.ico", "/favicon.png", "/apple-touch-icon.png"}
2726 discoverClient = &http.Client{Timeout: 15 * time.Second}
2827 )
2928
3029 func Discover(ctx context.Context, siteURL string) (*DiscoveryResult, error) {
31- result := &DiscoveryResult{}
32-
33- favicon := discoverFavicon(ctx, discoverClient, siteURL)
34- if favicon != "" {
35- result.Favicon = favicon
36- }
37-
38- feeds := discoverFeedLinks(ctx, discoverClient, siteURL)
39- result.FeedURLs = feeds
40-
41- return result, nil
42-}
43-
44-func discoverFeedLinks(ctx context.Context, client *http.Client, siteURL string) []string {
45- req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil)
46- if err != nil {
47- return nil
48- }
49- req.Header.Set("Accept", "text/html")
50-
51- resp, err := client.Do(req)
52- if err != nil {
53- return nil
54- }
55- defer resp.Body.Close()
56-
57- if resp.StatusCode != http.StatusOK {
58- return nil
30+ base, html := fetchHTML(ctx, siteURL)
31+ if base == nil {
32+ return &DiscoveryResult{}, nil
5933 }
6034
61- body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512))
62- if err != nil {
63- return nil
35+ if m := baseHrefRe.FindStringSubmatch(html); len(m) >= 2 && m[1] != "" {
36+ if u, err := base.Parse(m[1]); err == nil {
37+ base = u
38+ }
6439 }
6540
66- html := string(body)
67- baseURL := resolveBaseURL(siteURL, html)
41+ links := linkRe.FindAllString(html, -1)
42+ return &DiscoveryResult{
43+ FeedURLs: findFeedURLs(base, links),
44+ Favicon: findFavicon(ctx, base, links),
45+ }, nil
46+}
6847
48+func findFeedURLs(base *url.URL, links []string) []string {
6949 var feeds []string
70- links := linkRe.FindAllString(html, -1)
7150 for _, link := range links {
7251 if !relFeedRe.MatchString(link) && !typeFeedRe.MatchString(link) {
7352 continue
7453 }
75-
76- hrefMatch := hrefRe.FindStringSubmatch(link)
77- if len(hrefMatch) < 2 {
54+ href := extractHref(link)
55+ if href == "" {
7856 continue
7957 }
80-
81- feedURL := resolveURL(baseURL, hrefMatch[1])
82- if feedURL != "" {
83- feeds = append(feeds, feedURL)
58+ if u, err := base.Parse(href); err == nil {
59+ feeds = append(feeds, u.String())
8460 }
8561 }
86-
8762 return feeds
8863 }
8964
90-func discoverFavicon(ctx context.Context, client *http.Client, siteURL string) string {
91- req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil)
92- if err != nil {
93- return tryDefaultFavicons(ctx, client, siteURL)
94- }
95- req.Header.Set("Accept", "text/html")
96-
97- resp, err := client.Do(req)
98- if err != nil {
99- return tryDefaultFavicons(ctx, client, siteURL)
100- }
101- defer resp.Body.Close()
102-
103- if resp.StatusCode != http.StatusOK {
104- return tryDefaultFavicons(ctx, client, siteURL)
105- }
106-
107- body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512))
108- if err != nil {
109- return tryDefaultFavicons(ctx, client, siteURL)
110- }
111-
112- html := string(body)
113- baseURL := resolveBaseURL(siteURL, html)
114-
115- links := linkRe.FindAllString(html, -1)
65+func findFavicon(ctx context.Context, base *url.URL, links []string) string {
11666 for _, link := range links {
11767 if !relIconRe.MatchString(link) {
11868 continue
11969 }
120-
121- hrefMatch := hrefRe.FindStringSubmatch(link)
122- if len(hrefMatch) < 2 {
70+ href := extractHref(link)
71+ if href == "" {
12372 continue
12473 }
125-
126- iconURL := resolveURL(baseURL, hrefMatch[1])
127- if iconURL != "" {
128- return iconURL
74+ if u, err := base.Parse(href); err == nil {
75+ return u.String()
12976 }
13077 }
13178
132- return tryDefaultFavicons(ctx, client, siteURL)
133-}
134-
135-func tryDefaultFavicons(ctx context.Context, client *http.Client, siteURL string) string {
136- parsed := siteURL
137- if !strings.HasPrefix(parsed, "http") {
138- parsed = "https://" + parsed
139- }
140-
141- base := parsed
142- idx := strings.Index(base[8:], "/")
143- if idx >= 0 {
144- base = base[:8+idx]
145- }
79+ origin := *base
80+ origin.Path = ""
81+ origin.RawQuery = ""
82+ origin.Fragment = ""
14683
14784 for _, path := range faviconPaths {
148- url := base + path
149- req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
85+ u, _ := url.Parse(path)
86+ resolved := origin.ResolveReference(u)
87+ req, err := http.NewRequestWithContext(ctx, http.MethodHead, resolved.String(), nil)
15088 if err != nil {
15189 continue
15290 }
153-
154- resp, err := client.Do(req)
91+ resp, err := discoverClient.Do(req)
15592 if err != nil {
15693 continue
15794 }
15895 resp.Body.Close()
159-
16096 if resp.StatusCode == http.StatusOK {
161- return url
97+ return resolved.String()
16298 }
16399 }
164-
165100 return ""
166101 }
167102
168-func resolveBaseURL(siteURL, html string) string {
169- match := baseHrefRe.FindStringSubmatch(html)
170- if len(match) >= 2 && match[1] != "" {
171- return resolveURL(siteURL, match[1])
103+func extractHref(link string) string {
104+ m := hrefRe.FindStringSubmatch(link)
105+ if len(m) >= 2 {
106+ return m[1]
172107 }
173- return siteURL
108+ return ""
174109 }
175110
176-func resolveURL(base, ref string) string {
177- if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
178- return ref
111+func fetchHTML(ctx context.Context, siteURL string) (*url.URL, string) {
112+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil)
113+ if err != nil {
114+ return nil, ""
179115 }
116+ req.Header.Set("Accept", "text/html")
180117
181- base = strings.TrimRight(base, "/")
182- if strings.HasPrefix(ref, "//") {
183- return "https:" + ref
118+ resp, err := discoverClient.Do(req)
119+ if err != nil {
120+ return nil, ""
184121 }
185- if strings.HasPrefix(ref, "/") {
186- idx := strings.Index(base[8:], "/")
187- if idx >= 0 {
188- return base[:8+idx] + ref
189- }
190- return base + ref
122+ defer resp.Body.Close()
123+
124+ if resp.StatusCode != http.StatusOK {
125+ return nil, ""
191126 }
192127
193- idx := strings.LastIndex(base, "/")
194- if idx > 8 {
195- return base[:idx+1] + ref
128+ body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512))
129+ if err != nil {
130+ return nil, ""
196131 }
197- return fmt.Sprintf("%s/%s", base, ref)
132+
133+ base := resp.Request.URL
134+ return base, string(body)
198135 }
@@ -2,11 +2,10 @@ package feed
2 2
3 import (3 import (
4 "context"4 "context"
5- "fmt"
6 "io"5 "io"
7 "net/http"6 "net/http"
7+ "net/url"
8 "regexp"8 "regexp"
9- "strings"
10 "time"9 "time"
11 )10 )
12 11
@@ -16,183 +15,121 @@ type DiscoveryResult struct {
16 }15 }
17 16
18 var (17 var (
19- linkRe = regexp.MustCompile(`<link[^>]+>`)18+ linkRe = regexp.MustCompile(`<link[^>]+>`)
20- hrefRe = regexp.MustCompile(`href="([^"]*)"`)19+ hrefRe = regexp.MustCompile(`href="([^"]*)"`)
21- relFeedRe = regexp.MustCompile(`rel="(alternate|feed)"`)20+ relFeedRe = regexp.MustCompile(`rel="(alternate|feed)"`)
22- typeFeedRe = regexp.MustCompile(`type="([^"]*(?:rss|atom|feed|xml)[^"]*)"`)21+ typeFeedRe = regexp.MustCompile(`type="[^"]*(?:rss|atom|feed|xml)[^"]*"`)
23- relIconRe = regexp.MustCompile(`rel="[^"]*icon[^"]*"`)22+ relIconRe = regexp.MustCompile(`rel="[^"]*icon[^"]*"`)
24- baseHrefRe = regexp.MustCompile(`<base[^>]+href="([^"]*)"`)23+ baseHrefRe = regexp.MustCompile(`<base[^>]+href="([^"]*)"`)
25- faviconPaths = []string{"/favicon.ico", "/favicon.png", "/apple-touch-icon.png"}24+
26-25+ faviconPaths = []string{"/favicon.ico", "/favicon.png", "/apple-touch-icon.png"}
27 discoverClient = &http.Client{Timeout: 15 * time.Second}26 discoverClient = &http.Client{Timeout: 15 * time.Second}
28 )27 )
29 28
30 func Discover(ctx context.Context, siteURL string) (*DiscoveryResult, error) {29 func Discover(ctx context.Context, siteURL string) (*DiscoveryResult, error) {
31- result := &DiscoveryResult{}30+ base, html := fetchHTML(ctx, siteURL)
32-31+ if base == nil {
33- favicon := discoverFavicon(ctx, discoverClient, siteURL)32+ return &DiscoveryResult{}, nil
34- if favicon != "" {
35- result.Favicon = favicon
36- }
37-
38- feeds := discoverFeedLinks(ctx, discoverClient, siteURL)
39- result.FeedURLs = feeds
40-
41- return result, nil
42-}
43-
44-func discoverFeedLinks(ctx context.Context, client *http.Client, siteURL string) []string {
45- req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil)
46- if err != nil {
47- return nil
48- }
49- req.Header.Set("Accept", "text/html")
50-
51- resp, err := client.Do(req)
52- if err != nil {
53- return nil
54- }
55- defer resp.Body.Close()
56-
57- if resp.StatusCode != http.StatusOK {
58- return nil
59 }33 }
60 34
61- body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512))35+ if m := baseHrefRe.FindStringSubmatch(html); len(m) >= 2 && m[1] != "" {
62- if err != nil {36+ if u, err := base.Parse(m[1]); err == nil {
63- return nil37+ base = u
38+ }
64 }39 }
65 40
66- html := string(body)41+ links := linkRe.FindAllString(html, -1)
67- baseURL := resolveBaseURL(siteURL, html)42+ return &DiscoveryResult{
43+ FeedURLs: findFeedURLs(base, links),
44+ Favicon: findFavicon(ctx, base, links),
45+ }, nil
46+}
68 47
48+func findFeedURLs(base *url.URL, links []string) []string {
69 var feeds []string49 var feeds []string
70- links := linkRe.FindAllString(html, -1)
71 for _, link := range links {50 for _, link := range links {
72 if !relFeedRe.MatchString(link) && !typeFeedRe.MatchString(link) {51 if !relFeedRe.MatchString(link) && !typeFeedRe.MatchString(link) {
73 continue52 continue
74 }53 }
75-54+ href := extractHref(link)
76- hrefMatch := hrefRe.FindStringSubmatch(link)55+ if href == "" {
77- if len(hrefMatch) < 2 {
78 continue56 continue
79 }57 }
80-58+ if u, err := base.Parse(href); err == nil {
81- feedURL := resolveURL(baseURL, hrefMatch[1])59+ feeds = append(feeds, u.String())
82- if feedURL != "" {
83- feeds = append(feeds, feedURL)
84 }60 }
85 }61 }
86-
87 return feeds62 return feeds
88 }63 }
89 64
90-func discoverFavicon(ctx context.Context, client *http.Client, siteURL string) string {65+func findFavicon(ctx context.Context, base *url.URL, links []string) string {
91- req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil)
92- if err != nil {
93- return tryDefaultFavicons(ctx, client, siteURL)
94- }
95- req.Header.Set("Accept", "text/html")
96-
97- resp, err := client.Do(req)
98- if err != nil {
99- return tryDefaultFavicons(ctx, client, siteURL)
100- }
101- defer resp.Body.Close()
102-
103- if resp.StatusCode != http.StatusOK {
104- return tryDefaultFavicons(ctx, client, siteURL)
105- }
106-
107- body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512))
108- if err != nil {
109- return tryDefaultFavicons(ctx, client, siteURL)
110- }
111-
112- html := string(body)
113- baseURL := resolveBaseURL(siteURL, html)
114-
115- links := linkRe.FindAllString(html, -1)
116 for _, link := range links {66 for _, link := range links {
117 if !relIconRe.MatchString(link) {67 if !relIconRe.MatchString(link) {
118 continue68 continue
119 }69 }
120-70+ href := extractHref(link)
121- hrefMatch := hrefRe.FindStringSubmatch(link)71+ if href == "" {
122- if len(hrefMatch) < 2 {
123 continue72 continue
124 }73 }
125-74+ if u, err := base.Parse(href); err == nil {
126- iconURL := resolveURL(baseURL, hrefMatch[1])75+ return u.String()
127- if iconURL != "" {
128- return iconURL
129 }76 }
130 }77 }
131 78
132- return tryDefaultFavicons(ctx, client, siteURL)79+ origin := *base
133-}80+ origin.Path = ""
134-81+ origin.RawQuery = ""
135-func tryDefaultFavicons(ctx context.Context, client *http.Client, siteURL string) string {82+ origin.Fragment = ""
136- parsed := siteURL
137- if !strings.HasPrefix(parsed, "http") {
138- parsed = "https://" + parsed
139- }
140-
141- base := parsed
142- idx := strings.Index(base[8:], "/")
143- if idx >= 0 {
144- base = base[:8+idx]
145- }
146 83
147 for _, path := range faviconPaths {84 for _, path := range faviconPaths {
148- url := base + path85+ u, _ := url.Parse(path)
149- req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)86+ resolved := origin.ResolveReference(u)
87+ req, err := http.NewRequestWithContext(ctx, http.MethodHead, resolved.String(), nil)
150 if err != nil {88 if err != nil {
151 continue89 continue
152 }90 }
153-91+ resp, err := discoverClient.Do(req)
154- resp, err := client.Do(req)
155 if err != nil {92 if err != nil {
156 continue93 continue
157 }94 }
158 resp.Body.Close()95 resp.Body.Close()
159-
160 if resp.StatusCode == http.StatusOK {96 if resp.StatusCode == http.StatusOK {
161- return url97+ return resolved.String()
162 }98 }
163 }99 }
164-
165 return ""100 return ""
166 }101 }
167 102
168-func resolveBaseURL(siteURL, html string) string {103+func extractHref(link string) string {
169- match := baseHrefRe.FindStringSubmatch(html)104+ m := hrefRe.FindStringSubmatch(link)
170- if len(match) >= 2 && match[1] != "" {105+ if len(m) >= 2 {
171- return resolveURL(siteURL, match[1])106+ return m[1]
172 }107 }
173- return siteURL108+ return ""
174 }109 }
175 110
176-func resolveURL(base, ref string) string {111+func fetchHTML(ctx context.Context, siteURL string) (*url.URL, string) {
177- if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {112+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil)
178- return ref113+ if err != nil {
114+ return nil, ""
179 }115 }
116+ req.Header.Set("Accept", "text/html")
180 117
181- base = strings.TrimRight(base, "/")118+ resp, err := discoverClient.Do(req)
182- if strings.HasPrefix(ref, "//") {119+ if err != nil {
183- return "https:" + ref120+ return nil, ""
184 }121 }
185- if strings.HasPrefix(ref, "/") {122+ defer resp.Body.Close()
186- idx := strings.Index(base[8:], "/")123+
187- if idx >= 0 {124+ if resp.StatusCode != http.StatusOK {
188- return base[:8+idx] + ref125+ return nil, ""
189- }
190- return base + ref
191 }126 }
192 127
193- idx := strings.LastIndex(base, "/")128+ body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512))
194- if idx > 8 {129+ if err != nil {
195- return base[:idx+1] + ref130+ return nil, ""
196 }131 }
197- return fmt.Sprintf("%s/%s", base, ref)132+
133+ base := resp.Request.URL
134+ return base, string(body)
198 }135 }
modified internal/server/auth_handler.go +10 -55
@@ -3,11 +3,8 @@ package server
33 import (
44 "context"
55 "encoding/json"
6- "fmt"
76 "net/http"
8- "net/url"
97 "strings"
10- "os"
118
129 "github.com/bluesky-social/indigo/atproto/syntax"
1310
@@ -95,7 +92,15 @@ func (s *Server) handleOAuthCallback(w http.ResponseWriter, r *http.Request) {
9592 handle = ident.Handle.String()
9693 }
9794
98- displayName, avatarURL := s.fetchUserProfile(r.Context(), sessData)
95+ client := s.pdsClientFromSession(sessData)
96+
97+ var displayName, avatarURL string
98+ if client != nil {
99+ if dn, avatar, err := client.GetProfile(r.Context(), did); err == nil {
100+ displayName = dn
101+ avatarURL = avatar
102+ }
103+ }
99104
100105 user, err := s.db.CreateUser(r.Context(), did, handle, displayName, avatarURL)
101106 if err != nil {
@@ -125,33 +130,11 @@ func (s *Server) handleOAuthCallback(w http.ResponseWriter, r *http.Request) {
125130 SameSite: http.SameSiteLaxMode,
126131 })
127132
128- s.syncUserInBackground(user.DID, s.pdsClientFromSession(sessData))
133+ s.syncUserInBackground(user.DID, client)
129134
130135 http.Redirect(w, r, "/dashboard", http.StatusSeeOther)
131136 }
132137
133-func (s *Server) fetchUserProfile(ctx context.Context, sessData *oauth.ClientSessionData) (string, string) {
134- did := sessData.AccountDID.String()
135-
136- session, err := s.oauth.ResumeSession(ctx, sessData.AccountDID, sessData.SessionID)
137- if err != nil {
138- s.logger.Warn("failed to resume session for profile fetch", "error", err)
139- return "", ""
140- }
141-
142- var profile struct {
143- DisplayName string `json:"displayName"`
144- Avatar string `json:"avatar"`
145- }
146-
147- nsid, _ := syntax.ParseNSID("app.bsky.actor.getProfile")
148- if err := session.APIClient().Get(ctx, nsid, map[string]any{"actor": did}, &profile); err != nil {
149- s.logger.Warn("failed to fetch profile from PDS", "error", err, "did", did)
150- return "", ""
151- }
152- return profile.DisplayName, profile.Avatar
153-}
154-
155138 func (s *Server) pdsClientFromSession(sessData *oauth.ClientSessionData) *atproto.Client {
156139 session, err := s.oauth.ResumeSession(context.Background(), sessData.AccountDID, sessData.SessionID)
157140 if err != nil {
@@ -188,31 +171,3 @@ func (s *Server) handleAuthLogout(w http.ResponseWriter, r *http.Request) {
188171 s.clearUserSession(w)
189172 http.Redirect(w, r, "/", http.StatusSeeOther)
190173 }
191-
192-func resolveCallbackURL(r *http.Request) string {
193- scheme := "http"
194- if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
195- scheme = "https"
196- }
197- return fmt.Sprintf("%s://%s/auth/callback", scheme, r.Host)
198-}
199-
200-func resolveClientID(r *http.Request) string {
201- cid := os.Getenv("GLEAN_OAUTH_CLIENT_ID")
202- if cid != "" {
203- return cid
204- }
205- scheme := "http"
206- if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
207- scheme = "https"
208- }
209- return fmt.Sprintf("%s://%s/oauth/client-metadata", scheme, r.Host)
210-}
211-
212-func resolveBaseURL(r *http.Request) *url.URL {
213- scheme := "http"
214- if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
215- scheme = "https"
216- }
217- return &url.URL{Scheme: scheme, Host: r.Host}
218-}
@@ -3,11 +3,8 @@ package server
3 import (3 import (
4 "context"4 "context"
5 "encoding/json"5 "encoding/json"
6- "fmt"
7 "net/http"6 "net/http"
8- "net/url"
9 "strings"7 "strings"
10- "os"
11 8
12 "github.com/bluesky-social/indigo/atproto/syntax"9 "github.com/bluesky-social/indigo/atproto/syntax"
13 10
@@ -95,7 +92,15 @@ func (s *Server) handleOAuthCallback(w http.ResponseWriter, r *http.Request) {
95 handle = ident.Handle.String()92 handle = ident.Handle.String()
96 }93 }
97 94
98- displayName, avatarURL := s.fetchUserProfile(r.Context(), sessData)95+ client := s.pdsClientFromSession(sessData)
96+
97+ var displayName, avatarURL string
98+ if client != nil {
99+ if dn, avatar, err := client.GetProfile(r.Context(), did); err == nil {
100+ displayName = dn
101+ avatarURL = avatar
102+ }
103+ }
99 104
100 user, err := s.db.CreateUser(r.Context(), did, handle, displayName, avatarURL)105 user, err := s.db.CreateUser(r.Context(), did, handle, displayName, avatarURL)
101 if err != nil {106 if err != nil {
@@ -125,33 +130,11 @@ func (s *Server) handleOAuthCallback(w http.ResponseWriter, r *http.Request) {
125 SameSite: http.SameSiteLaxMode,130 SameSite: http.SameSiteLaxMode,
126 })131 })
127 132
128- s.syncUserInBackground(user.DID, s.pdsClientFromSession(sessData))133+ s.syncUserInBackground(user.DID, client)
129 134
130 http.Redirect(w, r, "/dashboard", http.StatusSeeOther)135 http.Redirect(w, r, "/dashboard", http.StatusSeeOther)
131 }136 }
132 137
133-func (s *Server) fetchUserProfile(ctx context.Context, sessData *oauth.ClientSessionData) (string, string) {
134- did := sessData.AccountDID.String()
135-
136- session, err := s.oauth.ResumeSession(ctx, sessData.AccountDID, sessData.SessionID)
137- if err != nil {
138- s.logger.Warn("failed to resume session for profile fetch", "error", err)
139- return "", ""
140- }
141-
142- var profile struct {
143- DisplayName string `json:"displayName"`
144- Avatar string `json:"avatar"`
145- }
146-
147- nsid, _ := syntax.ParseNSID("app.bsky.actor.getProfile")
148- if err := session.APIClient().Get(ctx, nsid, map[string]any{"actor": did}, &profile); err != nil {
149- s.logger.Warn("failed to fetch profile from PDS", "error", err, "did", did)
150- return "", ""
151- }
152- return profile.DisplayName, profile.Avatar
153-}
154-
155 func (s *Server) pdsClientFromSession(sessData *oauth.ClientSessionData) *atproto.Client {138 func (s *Server) pdsClientFromSession(sessData *oauth.ClientSessionData) *atproto.Client {
156 session, err := s.oauth.ResumeSession(context.Background(), sessData.AccountDID, sessData.SessionID)139 session, err := s.oauth.ResumeSession(context.Background(), sessData.AccountDID, sessData.SessionID)
157 if err != nil {140 if err != nil {
@@ -188,31 +171,3 @@ func (s *Server) handleAuthLogout(w http.ResponseWriter, r *http.Request) {
188 s.clearUserSession(w)171 s.clearUserSession(w)
189 http.Redirect(w, r, "/", http.StatusSeeOther)172 http.Redirect(w, r, "/", http.StatusSeeOther)
190 }173 }
191-
192-func resolveCallbackURL(r *http.Request) string {
193- scheme := "http"
194- if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
195- scheme = "https"
196- }
197- return fmt.Sprintf("%s://%s/auth/callback", scheme, r.Host)
198-}
199-
200-func resolveClientID(r *http.Request) string {
201- cid := os.Getenv("GLEAN_OAUTH_CLIENT_ID")
202- if cid != "" {
203- return cid
204- }
205- scheme := "http"
206- if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
207- scheme = "https"
208- }
209- return fmt.Sprintf("%s://%s/oauth/client-metadata", scheme, r.Host)
210-}
211-
212-func resolveBaseURL(r *http.Request) *url.URL {
213- scheme := "http"
214- if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
215- scheme = "https"
216- }
217- return &url.URL{Scheme: scheme, Host: r.Host}
218-}
modified internal/server/server.go +12 -1
@@ -490,7 +490,13 @@ func (s *Server) BackfillFromCollectionDir(ctx context.Context, collectionDirURL
490490 handle = ident.Handle.String()
491491 }
492492
493- if _, err := s.db.CreateUser(ctx, did, handle, "", ""); err != nil {
493+ var displayName, avatarURL string
494+ if _, dn, avatar, err := atproto.FetchProfile(ctx, did); err == nil {
495+ displayName = dn
496+ avatarURL = avatar
497+ }
498+
499+ if _, err := s.db.CreateUser(ctx, did, handle, displayName, avatarURL); err != nil {
494500 s.logger.Error("failed to create user during backfill", "error", err, "did", did)
495501 return
496502 }
@@ -545,6 +551,11 @@ func (s *Server) runSyncAll(ctx context.Context) {
545551 metrics.SyncErrors.Inc()
546552 s.logger.Error("periodic sync failed", "error", err, "did", u.DID)
547553 }
554+
555+ if dn, avatar, err := client.GetProfile(ctx, u.DID); err == nil {
556+ _ = s.db.UpdateUserProfile(ctx, u.DID, dn, avatar)
557+ }
558+
548559 metrics.SyncRuns.Inc()
549560 }
550561 }
@@ -490,7 +490,13 @@ func (s *Server) BackfillFromCollectionDir(ctx context.Context, collectionDirURL
490 handle = ident.Handle.String()490 handle = ident.Handle.String()
491 }491 }
492 492
493- if _, err := s.db.CreateUser(ctx, did, handle, "", ""); err != nil {493+ var displayName, avatarURL string
494+ if _, dn, avatar, err := atproto.FetchProfile(ctx, did); err == nil {
495+ displayName = dn
496+ avatarURL = avatar
497+ }
498+
499+ if _, err := s.db.CreateUser(ctx, did, handle, displayName, avatarURL); err != nil {
494 s.logger.Error("failed to create user during backfill", "error", err, "did", did)500 s.logger.Error("failed to create user during backfill", "error", err, "did", did)
495 return501 return
496 }502 }
@@ -545,6 +551,11 @@ func (s *Server) runSyncAll(ctx context.Context) {
545 metrics.SyncErrors.Inc()551 metrics.SyncErrors.Inc()
546 s.logger.Error("periodic sync failed", "error", err, "did", u.DID)552 s.logger.Error("periodic sync failed", "error", err, "did", u.DID)
547 }553 }
554+
555+ if dn, avatar, err := client.GetProfile(ctx, u.DID); err == nil {
556+ _ = s.db.UpdateUserProfile(ctx, u.DID, dn, avatar)
557+ }
558+
548 metrics.SyncRuns.Inc()559 metrics.SyncRuns.Inc()
549 }560 }
550 }561 }