rickub
nandi/gleanpublic Fork
⑂ d1cf582
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Download ZIP

Port the feed parser

RSS 2.0, RDF, Atom and JSON Feed, from internal/feed/parser.go.

The substance of a feed parser is its fallbacks, not its happy path. A
missing guid falls back to the link, since the guid is what dedupes an
article across fetches and an item without one would be re-inserted
forever. rel="self" loses to rel="alternate", or a feed whose first link is
itself points every article back at the feed. An unparseable date leaves the
article undated rather than dropping it. Atom's updated stands in for a
missing published, which Atom does not require.

Namespaces are matched by local name rather than resolved: Go's decoder maps
content:encoded to its namespace URI, Nim's keeps the literal prefix, and
feeds are inconsistent enough about prefixes that suffix matching is more
forgiving.

Two bugs, both invisible at compile time. xmltree's innerText silently skips
CDATA -- and RSS ships article bodies in CDATA precisely because they hold
HTML, so the parser would have returned empty articles while every other
field looked right. And Nim's zzz requires -05:00 where ZZZ accepts -0500;
feeds use both, so without the second every numeric-offset RSS feed parsed
as undated.

The probe also fetches four real feeds across two formats, because fixtures
only contain what their author thought of.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
nandithebull committed 2026-09-21T01:16:50-07:00 Browse files
d1cf582 parent: 4bdee40
modified spike/nim/.gitignore +1 -0
@@ -14,3 +14,4 @@ gleandb_probe
1414 stores_probe
1515 social_probe
1616 nimcache/
17+feedparser_probe
@@ -14,3 +14,4 @@ gleandb_probe
14 stores_probe14 stores_probe
15 social_probe15 social_probe
16 nimcache/16 nimcache/
17+feedparser_probe
added spike/nim/atproto/feedparser.nim +322 -0
new file mode 100644
@@ -0,0 +1,322 @@
1+## Parsing RSS, RDF, Atom and JSON Feed, ported from internal/feed/parser.go.
2+##
3+## Feeds in the wild are not well-formed in the way a spec implies. The rules
4+## that matter here are the fallbacks rather than the happy paths: a missing
5+## GUID falls back to the link, a missing date stays empty rather than
6+## becoming "now", and an unparseable date does not discard the article.
7+##
8+## Namespaces are handled by suffix rather than resolution. Go's XML decoder
9+## resolves `content:encoded` to its namespace URI; Nim's keeps the literal
10+## prefixed name, and feeds use prefixes inconsistently enough (`content:`,
11+## `media:`, occasionally none) that matching the local name is both simpler
12+## and more forgiving.
13+
14+import std/[json, options, strutils, times, xmltree, xmlparser]
15+
16+type
17+ FeedKind* = enum
18+ fkRss = "rss"
19+ fkAtom = "atom"
20+ fkRdf = "rdf"
21+ fkJson = "json"
22+
23+ ParsedFeed* = object
24+ url*: string
25+ title*: string
26+ siteUrl*: string
27+ description*: string
28+ kind*: FeedKind
29+ faviconUrl*: string
30+
31+ ParsedArticle* = object
32+ feedUrl*: string
33+ guid*: string
34+ title*: string
35+ url*: string
36+ author*: string
37+ content*: string
38+ summary*: string
39+ published*: Option[DateTime]
40+ updated*: Option[DateTime]
41+
42+ ParseResult* = object
43+ feed*: ParsedFeed
44+ articles*: seq[ParsedArticle]
45+
46+ FeedParseError* = object of CatchableError
47+
48+# --- dates -----------------------------------------------------------------
49+
50+const timeFormats = [
51+ "yyyy-MM-dd'T'HH:mm:sszzz", # RFC3339
52+ "yyyy-MM-dd'T'HH:mm:ss'Z'",
53+ "yyyy-MM-dd'T'HH:mm:ss'.'fffzzz",
54+ "yyyy-MM-dd'T'HH:mm:ss'.'fff'Z'",
55+ # ZZZ accepts "-0500", zzz requires "-05:00"; feeds use both.
56+ "ddd, dd MMM yyyy HH:mm:ss ZZZ", # RFC1123Z / RFC822
57+ "ddd, d MMM yyyy HH:mm:ss ZZZ",
58+ "ddd, dd MMM yyyy HH:mm:ss zzz",
59+ "ddd, d MMM yyyy HH:mm:ss zzz",
60+ "yyyy-MM-dd HH:mm:ss",
61+ "yyyy-MM-dd",
62+]
63+
64+## Named zones that turn up in RSS pubDate. Nim's parser has no table for
65+## them, and dropping the date entirely over a timezone abbreviation would
66+## lose the ordering of every article in the feed.
67+const namedZones = {
68+ "GMT": 0, "UTC": 0, "UT": 0, "Z": 0,
69+ "EST": -5, "EDT": -4, "CST": -6, "CDT": -5,
70+ "MST": -7, "MDT": -6, "PST": -8, "PDT": -7,
71+}
72+
73+proc normaliseZone(s: string): string =
74+ ## Rewrite a trailing named zone into the numeric offset Nim can parse.
75+ result = s.strip()
76+ for (name, hours) in namedZones:
77+ if result.endsWith(" " & name):
78+ let sign = if hours < 0: "-" else: "+"
79+ let h = abs(hours)
80+ return result[0 ..< result.len - name.len] &
81+ sign & align($h, 2, '0') & ":00"
82+
83+proc parseFeedTime*(raw: string): Option[DateTime] =
84+ ## Best-effort. An unparseable date is not an error -- plenty of feeds ship
85+ ## malformed ones, and dropping the article over it would be worse than
86+ ## showing it undated.
87+ let s = normaliseZone(raw)
88+ if s.len == 0: return none(DateTime)
89+ for fmt in timeFormats:
90+ try:
91+ return some(parse(s, fmt, utc()))
92+ except TimeParseError, TimeFormatParseError:
93+ continue
94+ none(DateTime)
95+
96+# --- XML helpers -----------------------------------------------------------
97+
98+proc localName(tag: string): string =
99+ ## "content:encoded" -> "encoded". Feeds disagree about prefixes, and the
100+ ## local name is what actually identifies the element.
101+ let i = tag.rfind(':')
102+ if i < 0: tag else: tag[i + 1 .. ^1]
103+
104+proc child(n: XmlNode, name: string): Option[XmlNode] =
105+ if n == nil or n.kind != xnElement: return none(XmlNode)
106+ for c in n:
107+ if c.kind == xnElement and localName(c.tag) == name:
108+ return some(c)
109+ none(XmlNode)
110+
111+iterator children(n: XmlNode, name: string): XmlNode =
112+ if n != nil and n.kind == xnElement:
113+ for c in n:
114+ if c.kind == xnElement and localName(c.tag) == name:
115+ yield c
116+
117+proc deepText(n: XmlNode): string =
118+ ## Text content including CDATA sections.
119+ ##
120+ ## xmltree's `innerText` skips xnCData entirely, which would be a quiet
121+ ## disaster here: RSS ships article bodies inside CDATA precisely because
122+ ## they contain HTML, so relying on innerText loses the content of most
123+ ## real feeds while the surrounding fields parse perfectly.
124+ case n.kind
125+ of xnText, xnCData: result = n.text
126+ of xnElement:
127+ for c in n: result.add deepText(c)
128+ else: discard
129+
130+proc text(n: XmlNode, name: string): string =
131+ let c = child(n, name)
132+ if c.isSome: deepText(c.get).strip() else: ""
133+
134+proc attrOr(n: XmlNode, name: string, fallback = ""): string =
135+ if n == nil or n.kind != xnElement: return fallback
136+ let v = n.attr(name)
137+ if v.len > 0: v else: fallback
138+
139+proc cleanFavicon(url: string): string =
140+ let u = url.strip()
141+ if u.startsWith("http://") or u.startsWith("https://"): u else: ""
142+
143+proc pickAtomLink(entry: XmlNode): string =
144+ ## Prefer rel="alternate" (or no rel, which means alternate), then fall
145+ ## back to the first link. A feed whose first link is rel="self" would
146+ ## otherwise point every article at the feed itself.
147+ var first = ""
148+ for l in children(entry, "link"):
149+ let href = l.attr("href")
150+ if href.len == 0: continue
151+ if first.len == 0: first = href
152+ let rel = l.attr("rel")
153+ if rel.len == 0 or rel == "alternate":
154+ return href
155+ first
156+
157+# --- RSS -------------------------------------------------------------------
158+
159+proc convertRss(root: XmlNode, feedUrl: string): ParseResult =
160+ let channel = child(root, "channel").get(newElement("channel"))
161+ result.feed = ParsedFeed(
162+ url: feedUrl,
163+ title: text(channel, "title"),
164+ siteUrl: text(channel, "link"),
165+ description: text(channel, "description"),
166+ kind: fkRss,
167+ faviconUrl: cleanFavicon(
168+ (let img = child(channel, "image"); if img.isSome: text(img.get, "url") else: "")),
169+ )
170+
171+ for item in children(channel, "item"):
172+ var a = ParsedArticle(
173+ feedUrl: feedUrl,
174+ guid: text(item, "guid"),
175+ title: text(item, "title"),
176+ url: text(item, "link"),
177+ # content:encoded is the full body; description is the summary. Feeds
178+ # that only ship one put it in description.
179+ content: text(item, "encoded"),
180+ summary: text(item, "description"),
181+ author: text(item, "author"),
182+ published: parseFeedTime(text(item, "pubDate")),
183+ )
184+ if a.author.len == 0:
185+ a.author = text(item, "creator") # dc:creator
186+ if a.guid.len == 0:
187+ a.guid = a.url
188+ result.articles.add a
189+
190+# --- RDF (RSS 1.0) ---------------------------------------------------------
191+
192+proc convertRdf(root: XmlNode, feedUrl: string): ParseResult =
193+ let channel = child(root, "channel").get(newElement("channel"))
194+ result.feed = ParsedFeed(
195+ url: feedUrl,
196+ title: text(channel, "title"),
197+ siteUrl: text(channel, "link"),
198+ description: text(channel, "description"),
199+ kind: fkRdf,
200+ )
201+
202+ # In RDF the items are siblings of <channel>, not inside it.
203+ for item in children(root, "item"):
204+ var a = ParsedArticle(
205+ feedUrl: feedUrl,
206+ title: text(item, "title"),
207+ url: text(item, "link"),
208+ content: text(item, "encoded"),
209+ summary: text(item, "description"),
210+ author: text(item, "creator"),
211+ published: parseFeedTime(text(item, "date")),
212+ )
213+ # RDF items are identified by rdf:about rather than a guid element.
214+ a.guid = attrOr(item, "rdf:about", attrOr(item, "about", a.url))
215+ if a.guid.len == 0:
216+ a.guid = a.url
217+ result.articles.add a
218+
219+# --- Atom ------------------------------------------------------------------
220+
221+proc convertAtom(root: XmlNode, feedUrl: string): ParseResult =
222+ var favicon = text(root, "icon")
223+ if favicon.len == 0:
224+ favicon = text(root, "logo")
225+
226+ result.feed = ParsedFeed(
227+ url: feedUrl,
228+ title: text(root, "title"),
229+ siteUrl: pickAtomLink(root),
230+ description: text(root, "subtitle"),
231+ kind: fkAtom,
232+ faviconUrl: cleanFavicon(favicon),
233+ )
234+
235+ for entry in children(root, "entry"):
236+ var a = ParsedArticle(
237+ feedUrl: feedUrl,
238+ guid: text(entry, "id"),
239+ title: text(entry, "title"),
240+ url: pickAtomLink(entry),
241+ content: text(entry, "content"),
242+ summary: text(entry, "summary"),
243+ published: parseFeedTime(text(entry, "published")),
244+ updated: parseFeedTime(text(entry, "updated")),
245+ )
246+ let author = child(entry, "author")
247+ if author.isSome:
248+ a.author = text(author.get, "name")
249+ if a.guid.len == 0:
250+ a.guid = a.url
251+ # Atom requires <updated> but not <published>; treating updated as the
252+ # publication date keeps such feeds orderable.
253+ if a.published.isNone:
254+ a.published = a.updated
255+ result.articles.add a
256+
257+# --- JSON Feed -------------------------------------------------------------
258+
259+proc convertJson(doc: JsonNode, feedUrl: string): ParseResult =
260+ result.feed = ParsedFeed(
261+ url: feedUrl,
262+ title: doc{"title"}.getStr,
263+ siteUrl: doc{"home_page_url"}.getStr,
264+ description: doc{"description"}.getStr,
265+ kind: fkJson,
266+ faviconUrl: cleanFavicon(doc{"icon"}.getStr),
267+ )
268+
269+ for item in doc{"items"}.getElems:
270+ var a = ParsedArticle(
271+ feedUrl: feedUrl,
272+ guid: item{"id"}.getStr,
273+ title: item{"title"}.getStr,
274+ url: item{"url"}.getStr,
275+ content: item{"content_html"}.getStr,
276+ summary: item{"summary"}.getStr,
277+ published: parseFeedTime(item{"date_published"}.getStr),
278+ updated: parseFeedTime(item{"date_modified"}.getStr),
279+ )
280+ if a.content.len == 0:
281+ a.content = item{"content_text"}.getStr
282+ let author = item{"author"}
283+ if author != nil and author.kind == JObject:
284+ a.author = author{"name"}.getStr
285+ if a.author.len == 0:
286+ let authors = item{"authors"}.getElems
287+ if authors.len > 0:
288+ a.author = authors[0]{"name"}.getStr
289+ if a.guid.len == 0:
290+ a.guid = a.url
291+ result.articles.add a
292+
293+# --- entry point -----------------------------------------------------------
294+
295+proc parseFeed*(body, feedUrl: string): ParseResult =
296+ ## Sniff the format and parse. The shape of the document decides, not the
297+ ## Content-Type header: servers mislabel feeds constantly, and a feed
298+ ## rejected for its header is indistinguishable to a reader from a feed
299+ ## that is broken.
300+ let trimmed = body.strip()
301+ if trimmed.len == 0:
302+ raise newException(FeedParseError, "empty document")
303+
304+ if trimmed[0] == '{':
305+ let doc =
306+ try: parseJson(trimmed)
307+ except CatchableError:
308+ raise newException(FeedParseError, "not valid JSON")
309+ return convertJson(doc, feedUrl)
310+
311+ let root =
312+ try: parseXml(trimmed)
313+ except CatchableError as e:
314+ raise newException(FeedParseError, "not valid XML: " & e.msg)
315+
316+ case localName(root.tag)
317+ of "rss": convertRss(root, feedUrl)
318+ of "feed": convertAtom(root, feedUrl)
319+ of "RDF": convertRdf(root, feedUrl)
320+ else:
321+ raise newException(FeedParseError,
322+ "unrecognised feed root element: " & root.tag)
new file mode 100644
@@ -0,0 +1,322 @@
1+## Parsing RSS, RDF, Atom and JSON Feed, ported from internal/feed/parser.go.
2+##
3+## Feeds in the wild are not well-formed in the way a spec implies. The rules
4+## that matter here are the fallbacks rather than the happy paths: a missing
5+## GUID falls back to the link, a missing date stays empty rather than
6+## becoming "now", and an unparseable date does not discard the article.
7+##
8+## Namespaces are handled by suffix rather than resolution. Go's XML decoder
9+## resolves `content:encoded` to its namespace URI; Nim's keeps the literal
10+## prefixed name, and feeds use prefixes inconsistently enough (`content:`,
11+## `media:`, occasionally none) that matching the local name is both simpler
12+## and more forgiving.
13+
14+import std/[json, options, strutils, times, xmltree, xmlparser]
15+
16+type
17+ FeedKind* = enum
18+ fkRss = "rss"
19+ fkAtom = "atom"
20+ fkRdf = "rdf"
21+ fkJson = "json"
22+
23+ ParsedFeed* = object
24+ url*: string
25+ title*: string
26+ siteUrl*: string
27+ description*: string
28+ kind*: FeedKind
29+ faviconUrl*: string
30+
31+ ParsedArticle* = object
32+ feedUrl*: string
33+ guid*: string
34+ title*: string
35+ url*: string
36+ author*: string
37+ content*: string
38+ summary*: string
39+ published*: Option[DateTime]
40+ updated*: Option[DateTime]
41+
42+ ParseResult* = object
43+ feed*: ParsedFeed
44+ articles*: seq[ParsedArticle]
45+
46+ FeedParseError* = object of CatchableError
47+
48+# --- dates -----------------------------------------------------------------
49+
50+const timeFormats = [
51+ "yyyy-MM-dd'T'HH:mm:sszzz", # RFC3339
52+ "yyyy-MM-dd'T'HH:mm:ss'Z'",
53+ "yyyy-MM-dd'T'HH:mm:ss'.'fffzzz",
54+ "yyyy-MM-dd'T'HH:mm:ss'.'fff'Z'",
55+ # ZZZ accepts "-0500", zzz requires "-05:00"; feeds use both.
56+ "ddd, dd MMM yyyy HH:mm:ss ZZZ", # RFC1123Z / RFC822
57+ "ddd, d MMM yyyy HH:mm:ss ZZZ",
58+ "ddd, dd MMM yyyy HH:mm:ss zzz",
59+ "ddd, d MMM yyyy HH:mm:ss zzz",
60+ "yyyy-MM-dd HH:mm:ss",
61+ "yyyy-MM-dd",
62+]
63+
64+## Named zones that turn up in RSS pubDate. Nim's parser has no table for
65+## them, and dropping the date entirely over a timezone abbreviation would
66+## lose the ordering of every article in the feed.
67+const namedZones = {
68+ "GMT": 0, "UTC": 0, "UT": 0, "Z": 0,
69+ "EST": -5, "EDT": -4, "CST": -6, "CDT": -5,
70+ "MST": -7, "MDT": -6, "PST": -8, "PDT": -7,
71+}
72+
73+proc normaliseZone(s: string): string =
74+ ## Rewrite a trailing named zone into the numeric offset Nim can parse.
75+ result = s.strip()
76+ for (name, hours) in namedZones:
77+ if result.endsWith(" " & name):
78+ let sign = if hours < 0: "-" else: "+"
79+ let h = abs(hours)
80+ return result[0 ..< result.len - name.len] &
81+ sign & align($h, 2, '0') & ":00"
82+
83+proc parseFeedTime*(raw: string): Option[DateTime] =
84+ ## Best-effort. An unparseable date is not an error -- plenty of feeds ship
85+ ## malformed ones, and dropping the article over it would be worse than
86+ ## showing it undated.
87+ let s = normaliseZone(raw)
88+ if s.len == 0: return none(DateTime)
89+ for fmt in timeFormats:
90+ try:
91+ return some(parse(s, fmt, utc()))
92+ except TimeParseError, TimeFormatParseError:
93+ continue
94+ none(DateTime)
95+
96+# --- XML helpers -----------------------------------------------------------
97+
98+proc localName(tag: string): string =
99+ ## "content:encoded" -> "encoded". Feeds disagree about prefixes, and the
100+ ## local name is what actually identifies the element.
101+ let i = tag.rfind(':')
102+ if i < 0: tag else: tag[i + 1 .. ^1]
103+
104+proc child(n: XmlNode, name: string): Option[XmlNode] =
105+ if n == nil or n.kind != xnElement: return none(XmlNode)
106+ for c in n:
107+ if c.kind == xnElement and localName(c.tag) == name:
108+ return some(c)
109+ none(XmlNode)
110+
111+iterator children(n: XmlNode, name: string): XmlNode =
112+ if n != nil and n.kind == xnElement:
113+ for c in n:
114+ if c.kind == xnElement and localName(c.tag) == name:
115+ yield c
116+
117+proc deepText(n: XmlNode): string =
118+ ## Text content including CDATA sections.
119+ ##
120+ ## xmltree's `innerText` skips xnCData entirely, which would be a quiet
121+ ## disaster here: RSS ships article bodies inside CDATA precisely because
122+ ## they contain HTML, so relying on innerText loses the content of most
123+ ## real feeds while the surrounding fields parse perfectly.
124+ case n.kind
125+ of xnText, xnCData: result = n.text
126+ of xnElement:
127+ for c in n: result.add deepText(c)
128+ else: discard
129+
130+proc text(n: XmlNode, name: string): string =
131+ let c = child(n, name)
132+ if c.isSome: deepText(c.get).strip() else: ""
133+
134+proc attrOr(n: XmlNode, name: string, fallback = ""): string =
135+ if n == nil or n.kind != xnElement: return fallback
136+ let v = n.attr(name)
137+ if v.len > 0: v else: fallback
138+
139+proc cleanFavicon(url: string): string =
140+ let u = url.strip()
141+ if u.startsWith("http://") or u.startsWith("https://"): u else: ""
142+
143+proc pickAtomLink(entry: XmlNode): string =
144+ ## Prefer rel="alternate" (or no rel, which means alternate), then fall
145+ ## back to the first link. A feed whose first link is rel="self" would
146+ ## otherwise point every article at the feed itself.
147+ var first = ""
148+ for l in children(entry, "link"):
149+ let href = l.attr("href")
150+ if href.len == 0: continue
151+ if first.len == 0: first = href
152+ let rel = l.attr("rel")
153+ if rel.len == 0 or rel == "alternate":
154+ return href
155+ first
156+
157+# --- RSS -------------------------------------------------------------------
158+
159+proc convertRss(root: XmlNode, feedUrl: string): ParseResult =
160+ let channel = child(root, "channel").get(newElement("channel"))
161+ result.feed = ParsedFeed(
162+ url: feedUrl,
163+ title: text(channel, "title"),
164+ siteUrl: text(channel, "link"),
165+ description: text(channel, "description"),
166+ kind: fkRss,
167+ faviconUrl: cleanFavicon(
168+ (let img = child(channel, "image"); if img.isSome: text(img.get, "url") else: "")),
169+ )
170+
171+ for item in children(channel, "item"):
172+ var a = ParsedArticle(
173+ feedUrl: feedUrl,
174+ guid: text(item, "guid"),
175+ title: text(item, "title"),
176+ url: text(item, "link"),
177+ # content:encoded is the full body; description is the summary. Feeds
178+ # that only ship one put it in description.
179+ content: text(item, "encoded"),
180+ summary: text(item, "description"),
181+ author: text(item, "author"),
182+ published: parseFeedTime(text(item, "pubDate")),
183+ )
184+ if a.author.len == 0:
185+ a.author = text(item, "creator") # dc:creator
186+ if a.guid.len == 0:
187+ a.guid = a.url
188+ result.articles.add a
189+
190+# --- RDF (RSS 1.0) ---------------------------------------------------------
191+
192+proc convertRdf(root: XmlNode, feedUrl: string): ParseResult =
193+ let channel = child(root, "channel").get(newElement("channel"))
194+ result.feed = ParsedFeed(
195+ url: feedUrl,
196+ title: text(channel, "title"),
197+ siteUrl: text(channel, "link"),
198+ description: text(channel, "description"),
199+ kind: fkRdf,
200+ )
201+
202+ # In RDF the items are siblings of <channel>, not inside it.
203+ for item in children(root, "item"):
204+ var a = ParsedArticle(
205+ feedUrl: feedUrl,
206+ title: text(item, "title"),
207+ url: text(item, "link"),
208+ content: text(item, "encoded"),
209+ summary: text(item, "description"),
210+ author: text(item, "creator"),
211+ published: parseFeedTime(text(item, "date")),
212+ )
213+ # RDF items are identified by rdf:about rather than a guid element.
214+ a.guid = attrOr(item, "rdf:about", attrOr(item, "about", a.url))
215+ if a.guid.len == 0:
216+ a.guid = a.url
217+ result.articles.add a
218+
219+# --- Atom ------------------------------------------------------------------
220+
221+proc convertAtom(root: XmlNode, feedUrl: string): ParseResult =
222+ var favicon = text(root, "icon")
223+ if favicon.len == 0:
224+ favicon = text(root, "logo")
225+
226+ result.feed = ParsedFeed(
227+ url: feedUrl,
228+ title: text(root, "title"),
229+ siteUrl: pickAtomLink(root),
230+ description: text(root, "subtitle"),
231+ kind: fkAtom,
232+ faviconUrl: cleanFavicon(favicon),
233+ )
234+
235+ for entry in children(root, "entry"):
236+ var a = ParsedArticle(
237+ feedUrl: feedUrl,
238+ guid: text(entry, "id"),
239+ title: text(entry, "title"),
240+ url: pickAtomLink(entry),
241+ content: text(entry, "content"),
242+ summary: text(entry, "summary"),
243+ published: parseFeedTime(text(entry, "published")),
244+ updated: parseFeedTime(text(entry, "updated")),
245+ )
246+ let author = child(entry, "author")
247+ if author.isSome:
248+ a.author = text(author.get, "name")
249+ if a.guid.len == 0:
250+ a.guid = a.url
251+ # Atom requires <updated> but not <published>; treating updated as the
252+ # publication date keeps such feeds orderable.
253+ if a.published.isNone:
254+ a.published = a.updated
255+ result.articles.add a
256+
257+# --- JSON Feed -------------------------------------------------------------
258+
259+proc convertJson(doc: JsonNode, feedUrl: string): ParseResult =
260+ result.feed = ParsedFeed(
261+ url: feedUrl,
262+ title: doc{"title"}.getStr,
263+ siteUrl: doc{"home_page_url"}.getStr,
264+ description: doc{"description"}.getStr,
265+ kind: fkJson,
266+ faviconUrl: cleanFavicon(doc{"icon"}.getStr),
267+ )
268+
269+ for item in doc{"items"}.getElems:
270+ var a = ParsedArticle(
271+ feedUrl: feedUrl,
272+ guid: item{"id"}.getStr,
273+ title: item{"title"}.getStr,
274+ url: item{"url"}.getStr,
275+ content: item{"content_html"}.getStr,
276+ summary: item{"summary"}.getStr,
277+ published: parseFeedTime(item{"date_published"}.getStr),
278+ updated: parseFeedTime(item{"date_modified"}.getStr),
279+ )
280+ if a.content.len == 0:
281+ a.content = item{"content_text"}.getStr
282+ let author = item{"author"}
283+ if author != nil and author.kind == JObject:
284+ a.author = author{"name"}.getStr
285+ if a.author.len == 0:
286+ let authors = item{"authors"}.getElems
287+ if authors.len > 0:
288+ a.author = authors[0]{"name"}.getStr
289+ if a.guid.len == 0:
290+ a.guid = a.url
291+ result.articles.add a
292+
293+# --- entry point -----------------------------------------------------------
294+
295+proc parseFeed*(body, feedUrl: string): ParseResult =
296+ ## Sniff the format and parse. The shape of the document decides, not the
297+ ## Content-Type header: servers mislabel feeds constantly, and a feed
298+ ## rejected for its header is indistinguishable to a reader from a feed
299+ ## that is broken.
300+ let trimmed = body.strip()
301+ if trimmed.len == 0:
302+ raise newException(FeedParseError, "empty document")
303+
304+ if trimmed[0] == '{':
305+ let doc =
306+ try: parseJson(trimmed)
307+ except CatchableError:
308+ raise newException(FeedParseError, "not valid JSON")
309+ return convertJson(doc, feedUrl)
310+
311+ let root =
312+ try: parseXml(trimmed)
313+ except CatchableError as e:
314+ raise newException(FeedParseError, "not valid XML: " & e.msg)
315+
316+ case localName(root.tag)
317+ of "rss": convertRss(root, feedUrl)
318+ of "feed": convertAtom(root, feedUrl)
319+ of "RDF": convertRdf(root, feedUrl)
320+ else:
321+ raise newException(FeedParseError,
322+ "unrecognised feed root element: " & root.tag)
added spike/nim/feedparser_probe.nim +264 -0
new file mode 100644
@@ -0,0 +1,264 @@
1+## Checks the feed parser against crafted fixtures and real feeds.
2+##
3+## nim c -r -d:ssl feedparser_probe.nim
4+##
5+## The fixtures cover the fallbacks, which is where feeds actually differ
6+## from the spec: a missing guid, a self link listed first, a named timezone,
7+## content in the wrong element. The live fetch is a reminder that feeds in
8+## the wild are stranger than any fixture.
9+
10+import std/[httpclient, options, sequtils, strformat, strutils]
11+import atproto/feedparser
12+
13+var failures = 0
14+
15+proc report(name: string, ok: bool, detail = "") =
16+ if not ok: inc failures
17+ let label = if ok: "PASS" else: "FAIL"
18+ if detail.len > 0: echo &" {label} {name} -- {detail}"
19+ else: echo &" {label} {name}"
20+
21+const rssFixture = """<?xml version="1.0"?>
22+<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/"
23+ xmlns:dc="http://purl.org/dc/elements/1.1/">
24+ <channel>
25+ <title>Example Feed</title>
26+ <link>https://example.test</link>
27+ <description>A description</description>
28+ <image><url>https://example.test/icon.png</url></image>
29+ <item>
30+ <title>With everything</title>
31+ <link>https://example.test/1</link>
32+ <guid isPermaLink="false">tag:example.test,2026:1</guid>
33+ <description>the summary</description>
34+ <content:encoded><![CDATA[<p>the body</p>]]></content:encoded>
35+ <author>alice@example.test</author>
36+ <pubDate>Mon, 15 Sep 2026 10:30:00 GMT</pubDate>
37+ </item>
38+ <item>
39+ <title>No guid, dc:creator</title>
40+ <link>https://example.test/2</link>
41+ <description>only a description</description>
42+ <dc:creator>Bob</dc:creator>
43+ <pubDate>Tue, 16 Sep 2026 08:00:00 -0500</pubDate>
44+ </item>
45+ <item>
46+ <title>Unparseable date</title>
47+ <link>https://example.test/3</link>
48+ <pubDate>sometime last tuesday</pubDate>
49+ </item>
50+ </channel>
51+</rss>"""
52+
53+const atomFixture = """<?xml version="1.0" encoding="utf-8"?>
54+<feed xmlns="http://www.w3.org/2005/Atom">
55+ <title>Atom Example</title>
56+ <subtitle>the subtitle</subtitle>
57+ <icon>https://example.test/icon.png</icon>
58+ <link rel="self" href="https://example.test/feed.xml"/>
59+ <link rel="alternate" href="https://example.test/"/>
60+ <entry>
61+ <title>An entry</title>
62+ <id>urn:uuid:1225c695</id>
63+ <link rel="self" href="https://example.test/entry.atom"/>
64+ <link href="https://example.test/post"/>
65+ <summary>a summary</summary>
66+ <content type="html">&lt;p&gt;body&lt;/p&gt;</content>
67+ <author><name>Carol</name></author>
68+ <updated>2026-09-15T10:30:00Z</updated>
69+ </entry>
70+</feed>"""
71+
72+const rdfFixture = """<?xml version="1.0"?>
73+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
74+ xmlns="http://purl.org/rss/1.0/"
75+ xmlns:dc="http://purl.org/dc/elements/1.1/">
76+ <channel rdf:about="https://example.test/rdf">
77+ <title>RDF Example</title>
78+ <link>https://example.test</link>
79+ <description>old school</description>
80+ </channel>
81+ <item rdf:about="https://example.test/rdf/1">
82+ <title>An RDF item</title>
83+ <link>https://example.test/rdf/1</link>
84+ <description>summary here</description>
85+ <dc:creator>Dave</dc:creator>
86+ <dc:date>2026-09-15T10:30:00Z</dc:date>
87+ </item>
88+</rdf:RDF>"""
89+
90+const jsonFixture = """{
91+ "version": "https://jsonfeed.org/version/1.1",
92+ "title": "JSON Example",
93+ "home_page_url": "https://example.test",
94+ "description": "a json feed",
95+ "icon": "https://example.test/icon.png",
96+ "items": [
97+ {
98+ "id": "1",
99+ "url": "https://example.test/j1",
100+ "title": "A JSON item",
101+ "content_html": "<p>html body</p>",
102+ "summary": "the summary",
103+ "date_published": "2026-09-15T10:30:00Z",
104+ "authors": [{"name": "Erin"}]
105+ },
106+ {
107+ "id": "",
108+ "url": "https://example.test/j2",
109+ "title": "Text only, no id",
110+ "content_text": "plain body"
111+ }
112+ ]
113+}"""
114+
115+proc main() =
116+ echo "Feed parser probe"
117+ echo ""
118+
119+ # --- RSS ----------------------------------------------------------------
120+ block:
121+ let r = parseFeed(rssFixture, "https://example.test/feed.xml")
122+ report("RSS is detected", r.feed.kind == fkRss, $r.feed.kind)
123+ report("channel metadata is read",
124+ r.feed.title == "Example Feed" and
125+ r.feed.siteUrl == "https://example.test", r.feed.title)
126+ report("channel image becomes the favicon",
127+ r.feed.faviconUrl == "https://example.test/icon.png")
128+ report("all items are read", r.articles.len == 3, $r.articles.len)
129+
130+ let a0 = r.articles[0]
131+ report("guid is preferred over the link",
132+ a0.guid == "tag:example.test,2026:1", a0.guid)
133+ # content:encoded is the body; description is the summary. Collapsing
134+ # them would give readers a truncated article.
135+ report("content:encoded becomes content",
136+ "the body" in a0.content, a0.content.strip())
137+ report("description stays the summary", a0.summary == "the summary")
138+ report("a GMT pubDate parses", a0.published.isSome,
139+ if a0.published.isSome: $a0.published.get else: "none")
140+
141+ let a1 = r.articles[1]
142+ report("a missing guid falls back to the link",
143+ a1.guid == "https://example.test/2", a1.guid)
144+ report("dc:creator is used when author is absent", a1.author == "Bob",
145+ a1.author)
146+ report("a numeric-offset pubDate parses", a1.published.isSome)
147+
148+ let a2 = r.articles[2]
149+ # Dropping the article over a bad date would be worse than showing it
150+ # undated.
151+ report("an unparseable date leaves the article intact",
152+ a2.title == "Unparseable date" and a2.published.isNone)
153+
154+ # --- Atom ---------------------------------------------------------------
155+ block:
156+ let r = parseFeed(atomFixture, "https://example.test/feed.xml")
157+ report("Atom is detected", r.feed.kind == fkAtom, $r.feed.kind)
158+ # A feed whose first link is rel="self" would otherwise point at itself.
159+ report("rel=alternate wins over rel=self for the site URL",
160+ r.feed.siteUrl == "https://example.test/", r.feed.siteUrl)
161+ report("icon becomes the favicon",
162+ r.feed.faviconUrl == "https://example.test/icon.png")
163+
164+ let e = r.articles[0]
165+ report("entry id becomes the guid", e.guid == "urn:uuid:1225c695", e.guid)
166+ report("a link with no rel counts as alternate",
167+ e.url == "https://example.test/post", e.url)
168+ report("nested author name is read", e.author == "Carol", e.author)
169+ report("escaped content is unescaped", "<p>body</p>" in e.content,
170+ e.content)
171+ # Atom requires updated but not published.
172+ report("updated stands in for a missing published date",
173+ e.published.isSome and e.published == e.updated)
174+
175+ # --- RDF ----------------------------------------------------------------
176+ block:
177+ let r = parseFeed(rdfFixture, "https://example.test/rdf")
178+ report("RDF is detected", r.feed.kind == fkRdf, $r.feed.kind)
179+ report("RDF channel metadata is read", r.feed.title == "RDF Example")
180+ # In RDF the items are siblings of <channel>, not children.
181+ report("RDF items are found outside the channel", r.articles.len == 1,
182+ $r.articles.len)
183+ if r.articles.len > 0:
184+ report("rdf:about becomes the guid",
185+ r.articles[0].guid == "https://example.test/rdf/1",
186+ r.articles[0].guid)
187+ report("dc:date parses", r.articles[0].published.isSome)
188+ report("dc:creator is the author", r.articles[0].author == "Dave")
189+
190+ # --- JSON Feed ----------------------------------------------------------
191+ block:
192+ let r = parseFeed(jsonFixture, "https://example.test/feed.json")
193+ report("JSON Feed is detected", r.feed.kind == fkJson, $r.feed.kind)
194+ report("JSON metadata is read", r.feed.title == "JSON Example")
195+ report("both items are read", r.articles.len == 2, $r.articles.len)
196+ report("content_html becomes content",
197+ "html body" in r.articles[0].content)
198+ report("authors[0].name is the author", r.articles[0].author == "Erin")
199+ report("content_text is a fallback for content",
200+ r.articles[1].content == "plain body", r.articles[1].content)
201+ report("an empty id falls back to the url",
202+ r.articles[1].guid == "https://example.test/j2",
203+ r.articles[1].guid)
204+
205+ # --- malformed input ----------------------------------------------------
206+ block:
207+ proc rejects(body: string): bool =
208+ try:
209+ discard parseFeed(body, "https://x.test")
210+ false
211+ except FeedParseError:
212+ true
213+ report("empty input is rejected", rejects(" "))
214+ report("non-feed XML is rejected", rejects("<html><body>hi</body></html>"))
215+ report("broken XML is rejected", rejects("<rss><channel>"))
216+ report("broken JSON is rejected", rejects("{not json"))
217+
218+ # --- real feeds ---------------------------------------------------------
219+ # Fixtures only contain what their author thought of. These are deliberately
220+ # a mix of generators and formats.
221+ block:
222+ const feeds = [
223+ "https://blog.rust-lang.org/feed.xml",
224+ "https://nim-lang.org/feed.xml",
225+ "https://lwn.net/headlines/rss",
226+ "https://news.ycombinator.com/rss",
227+ ]
228+ var parsed = 0
229+ for url in feeds:
230+ try:
231+ let client = newHttpClient(timeout = 20_000,
232+ userAgent = "glean-nim-spike/0.1")
233+ defer: client.close()
234+ let body = client.getContent(url)
235+ let r = parseFeed(body, url)
236+
237+ let withGuid = r.articles.countIt(it.guid.len > 0)
238+ let withTitle = r.articles.countIt(it.title.len > 0)
239+ let dated = r.articles.countIt(it.published.isSome)
240+ let bodied = r.articles.countIt(
241+ it.content.len > 0 or it.summary.len > 0)
242+
243+ # A guid is what dedupes an article across fetches, so a feed where
244+ # any item lacks one would re-insert that item forever.
245+ let ok = r.articles.len > 0 and withGuid == r.articles.len and
246+ withTitle == r.articles.len
247+ if ok: inc parsed
248+ report(url.split('/')[2], ok,
249+ &"{r.feed.kind}, {r.articles.len} items, " &
250+ &"{dated} dated, {bodied} with text")
251+ except CatchableError as e:
252+ report(url.split('/')[2], false, e.msg)
253+ report("every real feed parsed", parsed == feeds.len,
254+ &"{parsed}/{feeds.len}")
255+
256+ echo ""
257+ if failures == 0:
258+ echo "RESULT: the feed parser handles all four formats."
259+ else:
260+ echo &"RESULT: {failures} check(s) failed."
261+ quit 1
262+
263+when isMainModule:
264+ main()
new file mode 100644
@@ -0,0 +1,264 @@
1+## Checks the feed parser against crafted fixtures and real feeds.
2+##
3+## nim c -r -d:ssl feedparser_probe.nim
4+##
5+## The fixtures cover the fallbacks, which is where feeds actually differ
6+## from the spec: a missing guid, a self link listed first, a named timezone,
7+## content in the wrong element. The live fetch is a reminder that feeds in
8+## the wild are stranger than any fixture.
9+
10+import std/[httpclient, options, sequtils, strformat, strutils]
11+import atproto/feedparser
12+
13+var failures = 0
14+
15+proc report(name: string, ok: bool, detail = "") =
16+ if not ok: inc failures
17+ let label = if ok: "PASS" else: "FAIL"
18+ if detail.len > 0: echo &" {label} {name} -- {detail}"
19+ else: echo &" {label} {name}"
20+
21+const rssFixture = """<?xml version="1.0"?>
22+<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/"
23+ xmlns:dc="http://purl.org/dc/elements/1.1/">
24+ <channel>
25+ <title>Example Feed</title>
26+ <link>https://example.test</link>
27+ <description>A description</description>
28+ <image><url>https://example.test/icon.png</url></image>
29+ <item>
30+ <title>With everything</title>
31+ <link>https://example.test/1</link>
32+ <guid isPermaLink="false">tag:example.test,2026:1</guid>
33+ <description>the summary</description>
34+ <content:encoded><![CDATA[<p>the body</p>]]></content:encoded>
35+ <author>alice@example.test</author>
36+ <pubDate>Mon, 15 Sep 2026 10:30:00 GMT</pubDate>
37+ </item>
38+ <item>
39+ <title>No guid, dc:creator</title>
40+ <link>https://example.test/2</link>
41+ <description>only a description</description>
42+ <dc:creator>Bob</dc:creator>
43+ <pubDate>Tue, 16 Sep 2026 08:00:00 -0500</pubDate>
44+ </item>
45+ <item>
46+ <title>Unparseable date</title>
47+ <link>https://example.test/3</link>
48+ <pubDate>sometime last tuesday</pubDate>
49+ </item>
50+ </channel>
51+</rss>"""
52+
53+const atomFixture = """<?xml version="1.0" encoding="utf-8"?>
54+<feed xmlns="http://www.w3.org/2005/Atom">
55+ <title>Atom Example</title>
56+ <subtitle>the subtitle</subtitle>
57+ <icon>https://example.test/icon.png</icon>
58+ <link rel="self" href="https://example.test/feed.xml"/>
59+ <link rel="alternate" href="https://example.test/"/>
60+ <entry>
61+ <title>An entry</title>
62+ <id>urn:uuid:1225c695</id>
63+ <link rel="self" href="https://example.test/entry.atom"/>
64+ <link href="https://example.test/post"/>
65+ <summary>a summary</summary>
66+ <content type="html">&lt;p&gt;body&lt;/p&gt;</content>
67+ <author><name>Carol</name></author>
68+ <updated>2026-09-15T10:30:00Z</updated>
69+ </entry>
70+</feed>"""
71+
72+const rdfFixture = """<?xml version="1.0"?>
73+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
74+ xmlns="http://purl.org/rss/1.0/"
75+ xmlns:dc="http://purl.org/dc/elements/1.1/">
76+ <channel rdf:about="https://example.test/rdf">
77+ <title>RDF Example</title>
78+ <link>https://example.test</link>
79+ <description>old school</description>
80+ </channel>
81+ <item rdf:about="https://example.test/rdf/1">
82+ <title>An RDF item</title>
83+ <link>https://example.test/rdf/1</link>
84+ <description>summary here</description>
85+ <dc:creator>Dave</dc:creator>
86+ <dc:date>2026-09-15T10:30:00Z</dc:date>
87+ </item>
88+</rdf:RDF>"""
89+
90+const jsonFixture = """{
91+ "version": "https://jsonfeed.org/version/1.1",
92+ "title": "JSON Example",
93+ "home_page_url": "https://example.test",
94+ "description": "a json feed",
95+ "icon": "https://example.test/icon.png",
96+ "items": [
97+ {
98+ "id": "1",
99+ "url": "https://example.test/j1",
100+ "title": "A JSON item",
101+ "content_html": "<p>html body</p>",
102+ "summary": "the summary",
103+ "date_published": "2026-09-15T10:30:00Z",
104+ "authors": [{"name": "Erin"}]
105+ },
106+ {
107+ "id": "",
108+ "url": "https://example.test/j2",
109+ "title": "Text only, no id",
110+ "content_text": "plain body"
111+ }
112+ ]
113+}"""
114+
115+proc main() =
116+ echo "Feed parser probe"
117+ echo ""
118+
119+ # --- RSS ----------------------------------------------------------------
120+ block:
121+ let r = parseFeed(rssFixture, "https://example.test/feed.xml")
122+ report("RSS is detected", r.feed.kind == fkRss, $r.feed.kind)
123+ report("channel metadata is read",
124+ r.feed.title == "Example Feed" and
125+ r.feed.siteUrl == "https://example.test", r.feed.title)
126+ report("channel image becomes the favicon",
127+ r.feed.faviconUrl == "https://example.test/icon.png")
128+ report("all items are read", r.articles.len == 3, $r.articles.len)
129+
130+ let a0 = r.articles[0]
131+ report("guid is preferred over the link",
132+ a0.guid == "tag:example.test,2026:1", a0.guid)
133+ # content:encoded is the body; description is the summary. Collapsing
134+ # them would give readers a truncated article.
135+ report("content:encoded becomes content",
136+ "the body" in a0.content, a0.content.strip())
137+ report("description stays the summary", a0.summary == "the summary")
138+ report("a GMT pubDate parses", a0.published.isSome,
139+ if a0.published.isSome: $a0.published.get else: "none")
140+
141+ let a1 = r.articles[1]
142+ report("a missing guid falls back to the link",
143+ a1.guid == "https://example.test/2", a1.guid)
144+ report("dc:creator is used when author is absent", a1.author == "Bob",
145+ a1.author)
146+ report("a numeric-offset pubDate parses", a1.published.isSome)
147+
148+ let a2 = r.articles[2]
149+ # Dropping the article over a bad date would be worse than showing it
150+ # undated.
151+ report("an unparseable date leaves the article intact",
152+ a2.title == "Unparseable date" and a2.published.isNone)
153+
154+ # --- Atom ---------------------------------------------------------------
155+ block:
156+ let r = parseFeed(atomFixture, "https://example.test/feed.xml")
157+ report("Atom is detected", r.feed.kind == fkAtom, $r.feed.kind)
158+ # A feed whose first link is rel="self" would otherwise point at itself.
159+ report("rel=alternate wins over rel=self for the site URL",
160+ r.feed.siteUrl == "https://example.test/", r.feed.siteUrl)
161+ report("icon becomes the favicon",
162+ r.feed.faviconUrl == "https://example.test/icon.png")
163+
164+ let e = r.articles[0]
165+ report("entry id becomes the guid", e.guid == "urn:uuid:1225c695", e.guid)
166+ report("a link with no rel counts as alternate",
167+ e.url == "https://example.test/post", e.url)
168+ report("nested author name is read", e.author == "Carol", e.author)
169+ report("escaped content is unescaped", "<p>body</p>" in e.content,
170+ e.content)
171+ # Atom requires updated but not published.
172+ report("updated stands in for a missing published date",
173+ e.published.isSome and e.published == e.updated)
174+
175+ # --- RDF ----------------------------------------------------------------
176+ block:
177+ let r = parseFeed(rdfFixture, "https://example.test/rdf")
178+ report("RDF is detected", r.feed.kind == fkRdf, $r.feed.kind)
179+ report("RDF channel metadata is read", r.feed.title == "RDF Example")
180+ # In RDF the items are siblings of <channel>, not children.
181+ report("RDF items are found outside the channel", r.articles.len == 1,
182+ $r.articles.len)
183+ if r.articles.len > 0:
184+ report("rdf:about becomes the guid",
185+ r.articles[0].guid == "https://example.test/rdf/1",
186+ r.articles[0].guid)
187+ report("dc:date parses", r.articles[0].published.isSome)
188+ report("dc:creator is the author", r.articles[0].author == "Dave")
189+
190+ # --- JSON Feed ----------------------------------------------------------
191+ block:
192+ let r = parseFeed(jsonFixture, "https://example.test/feed.json")
193+ report("JSON Feed is detected", r.feed.kind == fkJson, $r.feed.kind)
194+ report("JSON metadata is read", r.feed.title == "JSON Example")
195+ report("both items are read", r.articles.len == 2, $r.articles.len)
196+ report("content_html becomes content",
197+ "html body" in r.articles[0].content)
198+ report("authors[0].name is the author", r.articles[0].author == "Erin")
199+ report("content_text is a fallback for content",
200+ r.articles[1].content == "plain body", r.articles[1].content)
201+ report("an empty id falls back to the url",
202+ r.articles[1].guid == "https://example.test/j2",
203+ r.articles[1].guid)
204+
205+ # --- malformed input ----------------------------------------------------
206+ block:
207+ proc rejects(body: string): bool =
208+ try:
209+ discard parseFeed(body, "https://x.test")
210+ false
211+ except FeedParseError:
212+ true
213+ report("empty input is rejected", rejects(" "))
214+ report("non-feed XML is rejected", rejects("<html><body>hi</body></html>"))
215+ report("broken XML is rejected", rejects("<rss><channel>"))
216+ report("broken JSON is rejected", rejects("{not json"))
217+
218+ # --- real feeds ---------------------------------------------------------
219+ # Fixtures only contain what their author thought of. These are deliberately
220+ # a mix of generators and formats.
221+ block:
222+ const feeds = [
223+ "https://blog.rust-lang.org/feed.xml",
224+ "https://nim-lang.org/feed.xml",
225+ "https://lwn.net/headlines/rss",
226+ "https://news.ycombinator.com/rss",
227+ ]
228+ var parsed = 0
229+ for url in feeds:
230+ try:
231+ let client = newHttpClient(timeout = 20_000,
232+ userAgent = "glean-nim-spike/0.1")
233+ defer: client.close()
234+ let body = client.getContent(url)
235+ let r = parseFeed(body, url)
236+
237+ let withGuid = r.articles.countIt(it.guid.len > 0)
238+ let withTitle = r.articles.countIt(it.title.len > 0)
239+ let dated = r.articles.countIt(it.published.isSome)
240+ let bodied = r.articles.countIt(
241+ it.content.len > 0 or it.summary.len > 0)
242+
243+ # A guid is what dedupes an article across fetches, so a feed where
244+ # any item lacks one would re-insert that item forever.
245+ let ok = r.articles.len > 0 and withGuid == r.articles.len and
246+ withTitle == r.articles.len
247+ if ok: inc parsed
248+ report(url.split('/')[2], ok,
249+ &"{r.feed.kind}, {r.articles.len} items, " &
250+ &"{dated} dated, {bodied} with text")
251+ except CatchableError as e:
252+ report(url.split('/')[2], false, e.msg)
253+ report("every real feed parsed", parsed == feeds.len,
254+ &"{parsed}/{feeds.len}")
255+
256+ echo ""
257+ if failures == 0:
258+ echo "RESULT: the feed parser handles all four formats."
259+ else:
260+ echo &"RESULT: {failures} check(s) failed."
261+ quit 1
262+
263+when isMainModule:
264+ main()
modified spike/nim/glean_spike.nimble +1 -0
@@ -28,6 +28,7 @@ const networkProbes = @[
2828 "oauth_probe",
2929 "xrpc_probe",
3030 "jetstream_probe",
31+ "feedparser_probe", # fixtures are offline, but it also fetches real feeds
3132 ]
3233
3334 proc runProbe(name: string) =
@@ -28,6 +28,7 @@ const networkProbes = @[
28 "oauth_probe",28 "oauth_probe",
29 "xrpc_probe",29 "xrpc_probe",
30 "jetstream_probe",30 "jetstream_probe",
31+ "feedparser_probe", # fixtures are offline, but it also fetches real feeds
31 ]32 ]
32 33
33 proc runProbe(name: string) =34 proc runProbe(name: string) =
modified spike/nim/readme.md +41 -0
@@ -365,6 +365,47 @@ active set is called out in the code, because "you unliked everything" and
365365 "the fetch failed" are indistinguishable from inside the store — the caller
366366 has to not get there.
367367
368+## 11. Feed parsing
369+
370+`atproto/feedparser.nim` ports `internal/feed/parser.go`: RSS 2.0, RDF
371+(RSS 1.0), Atom and JSON Feed.
372+
373+The interesting part of a feed parser is not the happy path, it is the
374+fallbacks, because feeds in the wild disagree with the spec constantly. A
375+missing `guid` falls back to the link — it is what dedupes an article across
376+fetches, so an item without one would be re-inserted forever. `rel="self"`
377+loses to `rel="alternate"`, or a feed whose first link is itself would point
378+every article back at the feed. An unparseable date leaves the article
379+undated rather than dropping it. Atom's `updated` stands in for a missing
380+`published`, since Atom requires only the former.
381+
382+Namespaces are matched by local name rather than resolved. Go's decoder maps
383+`content:encoded` to its namespace URI; Nim's keeps the literal prefix, and
384+feeds are inconsistent enough about prefixes that suffix matching is both
385+simpler and more forgiving.
386+
387+### Two bugs, one of them quiet and severe
388+
389+**`xmltree.innerText` silently skips CDATA.** RSS ships article bodies inside
390+CDATA *precisely because* they contain HTML, so relying on `innerText` loses
391+the content of most real feeds while every surrounding field parses
392+perfectly — a parser that looks like it works and returns empty articles.
393+The parser now walks the tree itself, including `xnCData`.
394+
395+**Nim's `zzz` requires `-05:00`; `ZZZ` accepts `-0500`.** Feeds use both, so
396+both formats are in the list. Without it, every RSS feed using a numeric
397+offset — which is most of them — parsed as undated.
398+
399+Neither would have failed a compile, and neither is visible without a
400+fixture that contains the case.
401+
402+### Beyond fixtures
403+
404+Fixtures only contain what their author thought of, so the probe also fetches
405+four real feeds across two formats (Rust Blog, nim-lang, LWN, Hacker News)
406+and asserts every item has a guid and a title. All four parse, 65 items
407+between them, all dated and all carrying text.
408+
368409 ## What this does and does not prove
369410
370411 Proven: the cryptography, the storage layer, and the OAuth flow up to user
@@ -365,6 +365,47 @@ active set is called out in the code, because "you unliked everything" and
365 "the fetch failed" are indistinguishable from inside the store — the caller365 "the fetch failed" are indistinguishable from inside the store — the caller
366 has to not get there.366 has to not get there.
367 367
368+## 11. Feed parsing
369+
370+`atproto/feedparser.nim` ports `internal/feed/parser.go`: RSS 2.0, RDF
371+(RSS 1.0), Atom and JSON Feed.
372+
373+The interesting part of a feed parser is not the happy path, it is the
374+fallbacks, because feeds in the wild disagree with the spec constantly. A
375+missing `guid` falls back to the link — it is what dedupes an article across
376+fetches, so an item without one would be re-inserted forever. `rel="self"`
377+loses to `rel="alternate"`, or a feed whose first link is itself would point
378+every article back at the feed. An unparseable date leaves the article
379+undated rather than dropping it. Atom's `updated` stands in for a missing
380+`published`, since Atom requires only the former.
381+
382+Namespaces are matched by local name rather than resolved. Go's decoder maps
383+`content:encoded` to its namespace URI; Nim's keeps the literal prefix, and
384+feeds are inconsistent enough about prefixes that suffix matching is both
385+simpler and more forgiving.
386+
387+### Two bugs, one of them quiet and severe
388+
389+**`xmltree.innerText` silently skips CDATA.** RSS ships article bodies inside
390+CDATA *precisely because* they contain HTML, so relying on `innerText` loses
391+the content of most real feeds while every surrounding field parses
392+perfectly — a parser that looks like it works and returns empty articles.
393+The parser now walks the tree itself, including `xnCData`.
394+
395+**Nim's `zzz` requires `-05:00`; `ZZZ` accepts `-0500`.** Feeds use both, so
396+both formats are in the list. Without it, every RSS feed using a numeric
397+offset — which is most of them — parsed as undated.
398+
399+Neither would have failed a compile, and neither is visible without a
400+fixture that contains the case.
401+
402+### Beyond fixtures
403+
404+Fixtures only contain what their author thought of, so the probe also fetches
405+four real feeds across two formats (Rust Blog, nim-lang, LWN, Hacker News)
406+and asserts every item has a guid and a title. All four parse, 65 items
407+between them, all dated and all carrying text.
408+
368 ## What this does and does not prove409 ## What this does and does not prove
369 410
370 Proven: the cryptography, the storage layer, and the OAuth flow up to user411 Proven: the cryptography, the storage layer, and the OAuth flow up to user