nandi/glean · 212dde3

modified internal/atproto/lexicon_external.go +25 -1 Viewed

@@ -3,7 +3,10 @@
3	3	// See lexicon_test.go for the test ensuring these stay in sync with lexicons/.
4	4	package atproto
5	5
6		-import "encoding/json"
	6	+import (
	7	+ "encoding/json"
	8	+ "strings"
	9	+)
7	10
8	11	type FollowRecord struct {
9	12	Subject string `json:"subject"`
@@ -144,3 +147,24 @@ type StandardDocumentRecord struct {
144	147	Links json.RawMessage `json:"links,omitempty"`
145	148	Tags []string `json:"tags,omitempty"`
146	149	}
	150	+
	151	+type StandardDocumentLink struct {
	152	+ Rel string `json:"rel"`
	153	+ URI string `json:"uri"`
	154	+}
	155	+
	156	+func (doc *StandardDocumentRecord) RelatedLinkURL() string {
	157	+ if len(doc.Links) == 0 {
	158	+ return ""
	159	+ }
	160	+ var links []StandardDocumentLink
	161	+ if err := json.Unmarshal(doc.Links, &links); err != nil {
	162	+ return ""
	163	+ }
	164	+ for _, l := range links {
	165	+ if l.Rel == "related" && strings.HasPrefix(l.URI, "http") {
	166	+ return l.URI
	167	+ }
	168	+ }
	169	+ return ""
	170	+}

  // See lexicon_test.go for the test ensuring these stay in sync with lexicons/.
  package atproto
--import "encoding/json"
++import (
++	"encoding/json"
++	"strings"
++)
  type FollowRecord struct {
  	Subject   string          `json:"subject"`
  	Links        json.RawMessage `json:"links,omitempty"`
  	Tags         []string        `json:"tags,omitempty"`
  }
++
++type StandardDocumentLink struct {
++	Rel string `json:"rel"`
++	URI string `json:"uri"`
++}
++
++func (doc *StandardDocumentRecord) RelatedLinkURL() string {
++	if len(doc.Links) == 0 {
++		return ""
++	}
++	var links []StandardDocumentLink
++	if err := json.Unmarshal(doc.Links, &links); err != nil {
++		return ""
++	}
++	for _, l := range links {
++		if l.Rel == "related" && strings.HasPrefix(l.URI, "http") {
++			return l.URI
++		}
++	}
++	return ""
++}

modified internal/atproto/standard_site_fetcher.go +6 -1 Viewed

@@ -91,11 +91,16 @@ func (f StandardSiteFetcher) fetchDocuments(ctx context.Context, client Client
91	91	published := parseRFC3339(doc.PublishedAt)
92	92	updated := parseRFC3339(doc.UpdatedAt)
93	93
	94	+ articleURL := publicationURL + doc.Path
	95	+ if related := doc.RelatedLinkURL(); related != "" && articleURL == "" {
	96	+ articleURL = related
	97	+ }
	98	+
94	99	articles = append(articles, feed.Article{
95	100	FeedURL: publicationURI,
96	101	GUID: r.URI,
97	102	Title: doc.Title,
98		- URL: publicationURL + doc.Path,
	103	+ URL: articleURL,
99	104	Author: firstContributor(doc.Contributors),
100	105	Content: doc.TextContent,
101	106	Summary: doc.Description,

  		published := parseRFC3339(doc.PublishedAt)
  		updated := parseRFC3339(doc.UpdatedAt)
++		articleURL := publicationURL + doc.Path
++		if related := doc.RelatedLinkURL(); related != "" && articleURL == "" {
++			articleURL = related
++		}
++
  		articles = append(articles, feed.Article{
  			FeedURL:   publicationURI,
  			GUID:      r.URI,
  			Title:     doc.Title,
--			URL:       publicationURL + doc.Path,
++			URL:       articleURL,
  			Author:    firstContributor(doc.Contributors),
  			Content:   doc.TextContent,
  			Summary:   doc.Description,

modified internal/atproto/stream_handler.go +3 -0 Viewed

@@ -311,6 +311,9 @@ func (h StreamDBHandler) handleStandardDocument(ctx context.Context, event Eve
311	311	if f, err := h.articles.GetFeed(ctx, publicationURI); err == nil && f.SiteURL.Valid {
312	312	articleURL = f.SiteURL.String + doc.Path
313	313	}
	314	+ if related := doc.RelatedLinkURL(); related != "" && articleURL == "" {
	315	+ articleURL = related
	316	+ }
314	317
315	318	articles := []feed.Article{{
316	319	FeedURL: publicationURI,

  		if f, err := h.articles.GetFeed(ctx, publicationURI); err == nil && f.SiteURL.Valid {
  			articleURL = f.SiteURL.String + doc.Path
  		}
++		if related := doc.RelatedLinkURL(); related != "" && articleURL == "" {
++			articleURL = related
++		}
  		articles := []feed.Article{{
  			FeedURL:   publicationURI,

feat: extract related link from standard.documentUnverified