nandi/glean · 60367de

nandi/gleanpublic⑂ Fork 0

⑂ 60367de

fix(atproto): resolve missing publication URLsUnverified

Julien Robert committed 2026-08-09T15:43:53+02:00 Browse files

60367de parent: fbad959

modified internal/atproto/standard_site_fetcher.go +27 -1 Viewed

@@ -92,7 +92,7 @@ func (f StandardSiteFetcher) fetchDocuments(ctx context.Context, client Client
92	92	updated := parseRFC3339(doc.UpdatedAt)
93	93
94	94	articleURL := publicationURL + doc.Path
95		- if related := doc.RelatedLinkURL(); related != "" && articleURL == "" {
	95	+ if related := doc.RelatedLinkURL(); related != "" && publicationURL == "" {
96	96	articleURL = related
97	97	}
98	98
@@ -149,3 +149,29 @@ func parseRFC3339(s string) time.Time {
149	149	t, _ := time.Parse(time.RFC3339, s)
150	150	return t
151	151	}
	152	+
	153	+// resolvePublicationURL returns the site URL for a standard.publication record.
	154	+// It fetches the publication record from the author's PDS when not cached.
	155	+func resolvePublicationURL(ctx context.Context, publicationURI string) (string, error) {
	156	+ parsed, ok := ParseRecordURI(publicationURI)
	157	+ if !ok {
	158	+ return "", fmt.Errorf("invalid publication URI: %s", publicationURI)
	159	+ }
	160	+
	161	+ pdsURL, err := ResolvePDSEndpoint(ctx, parsed.DID)
	162	+ if err != nil {
	163	+ return "", fmt.Errorf("resolving PDS: %w", err)
	164	+ }
	165	+
	166	+ client := NewUnauthenticatedClient(pdsURL)
	167	+ raw, err := client.GetRecord(ctx, parsed.DID, parsed.Collection, parsed.RKey)
	168	+ if err != nil {
	169	+ return "", fmt.Errorf("fetching publication record: %w", err)
	170	+ }
	171	+
	172	+ var pub StandardPublicationRecord
	173	+ if err := json.Unmarshal(raw, &pub); err != nil {
	174	+ return "", fmt.Errorf("parsing publication record: %w", err)
	175	+ }
	176	+ return pub.URL, nil
	177	+}

  		updated := parseRFC3339(doc.UpdatedAt)
  		articleURL := publicationURL + doc.Path
--		if related := doc.RelatedLinkURL(); related != "" && articleURL == "" {
++		if related := doc.RelatedLinkURL(); related != "" && publicationURL == "" {
  			articleURL = related
  		}
  	t, _ := time.Parse(time.RFC3339, s)
  	return t
  }
++
++// resolvePublicationURL returns the site URL for a standard.publication record.
++// It fetches the publication record from the author's PDS when not cached.
++func resolvePublicationURL(ctx context.Context, publicationURI string) (string, error) {
++	parsed, ok := ParseRecordURI(publicationURI)
++	if !ok {
++		return "", fmt.Errorf("invalid publication URI: %s", publicationURI)
++	}
++
++	pdsURL, err := ResolvePDSEndpoint(ctx, parsed.DID)
++	if err != nil {
++		return "", fmt.Errorf("resolving PDS: %w", err)
++	}
++
++	client := NewUnauthenticatedClient(pdsURL)
++	raw, err := client.GetRecord(ctx, parsed.DID, parsed.Collection, parsed.RKey)
++	if err != nil {
++		return "", fmt.Errorf("fetching publication record: %w", err)
++	}
++
++	var pub StandardPublicationRecord
++	if err := json.Unmarshal(raw, &pub); err != nil {
++		return "", fmt.Errorf("parsing publication record: %w", err)
++	}
++	return pub.URL, nil
++}

modified internal/atproto/stream_handler.go +17 -9 Viewed

@@ -302,16 +302,24 @@ func (h StreamDBHandler) handleStandardDocument(ctx context.Context, event Eve
302	302	published := parseRFC3339(doc.PublishedAt)
303	303	updated := parseRFC3339(doc.UpdatedAt)
304	304
305		- _ = h.articles.UpsertFeed(ctx, &db.Feed{
306		- FeedURL: publicationURI,
307		- FeedType: sql.NullString{String: "atproto", Valid: true},
308		- })
309		-
310		- var articleURL string
311		- if f, err := h.articles.GetFeed(ctx, publicationURI); err == nil && f.SiteURL.Valid {
312		- articleURL = f.SiteURL.String + doc.Path
	305	+ existingFeed, _ := h.articles.GetFeed(ctx, publicationURI)
	306	+
	307	+ publicationURL := ""
	308	+ if existingFeed != nil && existingFeed.SiteURL.Valid {
	309	+ publicationURL = existingFeed.SiteURL.String
	310	+ } else if resolved, err := resolvePublicationURL(ctx, publicationURI); err == nil && resolved != "" {
	311	+ publicationURL = resolved
	312	+ _ = h.articles.UpsertFeed(ctx, &db.Feed{
	313	+ FeedURL: publicationURI,
	314	+ SiteURL: db.NullStr(publicationURL),
	315	+ FeedType: sql.NullString{String: "atproto", Valid: true},
	316	+ })
	317	+ } else if err != nil {
	318	+ h.logger.Warn("failed to resolve publication URL", "error", err, "uri", publicationURI)
313	319	}
314		- if related := doc.RelatedLinkURL(); related != "" && articleURL == "" {
	320	+
	321	+ articleURL := publicationURL + doc.Path
	322	+ if related := doc.RelatedLinkURL(); related != "" && publicationURL == "" {
315	323	articleURL = related
316	324	}
317	325

  		published := parseRFC3339(doc.PublishedAt)
  		updated := parseRFC3339(doc.UpdatedAt)
--		_ = h.articles.UpsertFeed(ctx, &db.Feed{
++		existingFeed, _ := h.articles.GetFeed(ctx, publicationURI)
--			FeedURL:  publicationURI,
++
--			FeedType: sql.NullString{String: "atproto", Valid: true},
++		publicationURL := ""
--		})
++		if existingFeed != nil && existingFeed.SiteURL.Valid {
--
++			publicationURL = existingFeed.SiteURL.String
--		var articleURL string
++		} else if resolved, err := resolvePublicationURL(ctx, publicationURI); err == nil && resolved != "" {
--		if f, err := h.articles.GetFeed(ctx, publicationURI); err == nil && f.SiteURL.Valid {
++			publicationURL = resolved
--			articleURL = f.SiteURL.String + doc.Path
++			_ = h.articles.UpsertFeed(ctx, &db.Feed{
++				FeedURL:  publicationURI,
++				SiteURL:  db.NullStr(publicationURL),
++				FeedType: sql.NullString{String: "atproto", Valid: true},
++			})
++		} else if err != nil {
++			h.logger.Warn("failed to resolve publication URL", "error", err, "uri", publicationURI)
  		}
--		if related := doc.RelatedLinkURL(); related != "" && articleURL == "" {
++
++		articleURL := publicationURL + doc.Path
++		if related := doc.RelatedLinkURL(); related != "" && publicationURL == "" {
  			articleURL = related
  		}