rickub
nandi/gleanpublic Fork
⑂ e933815
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Download ZIP

Port the article scraper

Feeds routinely ship a first paragraph and a "read more" link, so the reader
view needs the page: find the node most likely to be the article, strip the
furniture, re-render a whitelisted subset of HTML. Output stays HTML rather
than text, since paragraphs, headings, lists and code blocks are most of
what makes a long article readable.

Sanitisation is the part worth testing. This renders HTML from an arbitrary
site into a reader's client, so the probe asserts what must not survive:
script and style content, iframes, event handlers, unknown data attributes,
javascript: URLs. Attributes are whitelisted rather than blacklisted, so one
nobody anticipated is dropped by default. Links carry rel=noopener, and a
link that does not go anywhere reachable is flattened to its text.

Two details that only show up in a fixture. A lazy-loaded image resolves
through src, data-src and data-lazy-src to the first reachable URL, because
rendering the placeholder puts a broken image where the picture goes. And
removal walks children back to front: deleting by index shifts the siblings
after it, so a forward walk skips one per removal -- the probe has four
consecutive nav elements to catch exactly that.

The extractor refuses rather than returning something thin, since a caller
handed a nav menu cannot tell it from a short article.

The live check derives its URL from the Rust blog's feed after a hard-coded
guess 404'd; a probe that rots when someone reorganises permalinks is worse
than no probe.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
nandithebull committed 2026-09-21T18:35:26-07:00 Browse files
e933815 parent: fdcbe24
modified spike/nim/.gitignore +1 -0
@@ -16,3 +16,4 @@ social_probe
1616 nimcache/
1717 feedparser_probe
1818 feedfetcher_probe
19+scraper_probe
@@ -16,3 +16,4 @@ social_probe
16 nimcache/16 nimcache/
17 feedparser_probe17 feedparser_probe
18 feedfetcher_probe18 feedfetcher_probe
19+scraper_probe
added spike/nim/atproto/scraper.nim +285 -0
new file mode 100644
@@ -0,0 +1,285 @@
1+## Extracting an article from a page, ported from internal/scraper/scraper.go.
2+##
3+## Feeds routinely ship a first paragraph and a "read more" link, so the
4+## reader view needs the page itself. This is a readability-style extractor:
5+## find the node most likely to be the article, strip the furniture around
6+## it, and re-render a small whitelisted subset of HTML.
7+##
8+## Two things are deliberate:
9+##
10+## * The output is sanitised HTML, not text. Paragraphs, headings, lists,
11+## code blocks and images are what make a long article readable, and
12+## flattening them to text loses more than the markup costs.
13+## * Nothing is trusted. Attributes are whitelisted rather than blacklisted,
14+## and `script`, `style`, `iframe` and friends never survive, because this
15+## renders HTML fetched from an arbitrary site into the reader's client.
16+
17+import std/[htmlparser, options, strtabs, strutils, xmltree]
18+
19+type
20+ ScrapeError* = object of CatchableError
21+
22+const
23+ MinContentLength* = 200
24+ ## Below this a candidate is furniture, not an article.
25+
26+ ## Ordered: the first that matches wins. Semantic markup beats guessing,
27+ ## so <article> and role=main come before class-name heuristics.
28+ ContentPatterns = [
29+ "post-content", "entry-content", "article-content",
30+ "article-body", "article__body", "article__content",
31+ "story-body", "story-content", "content-body",
32+ "post-body", "post-entry", "entry-body",
33+ "blog-content", "blog-post", "wp-content",
34+ "main-content", "page-content", "body-content",
35+ ]
36+
37+ UnwantedTags = [
38+ "script", "style", "nav", "header", "footer", "aside", "noscript",
39+ "iframe", "form", "svg", "button", "input", "textarea", "select",
40+ ]
41+
42+ UnwantedRoles = ["complementary", "banner", "contentinfo", "navigation"]
43+
44+ UnwantedClassPatterns = [
45+ "comment", "sidebar", "advertisement", "ad-banner",
46+ "social-share", "share-button", "newsletter", "popup",
47+ "cookie", "paywall", "related-post", "related-article",
48+ "taboola", "outbrain", "disqus",
49+ ]
50+
51+ ## Narrower than the class list: an id of "header" is reliably the page
52+ ## header, while a *class* of "header" is often the article's own.
53+ UnwantedIdPatterns = ["comment", "sidebar", "footer", "header", "nav", "disqus"]
54+
55+ VoidElements = [
56+ "br", "hr", "img", "input", "meta", "link", "area", "base", "col",
57+ "embed", "source", "track", "wbr",
58+ ]
59+
60+ AllowedAttrs = [
61+ "href", "src", "alt", "title", "class", "id", "width", "height",
62+ "type", "controls", "preload", "poster", "cite", "datetime",
63+ "colspan", "rowspan", "loading", "decoding", "itemprop",
64+ "role", "aria-label", "aria-hidden",
65+ ]
66+
67+ ## Images keep only presentational attributes; `src` is resolved separately
68+ ## so that lazy-loading placeholders do not survive as broken images.
69+ ImgAllowedAttrs = ["alt", "width", "height", "class"]
70+
71+proc attrOf(n: XmlNode, key: string): string =
72+ if n.kind != xnElement or n.attrs == nil: "" else: n.attrs.getOrDefault(key)
73+
74+proc containsAny(s: string, patterns: openArray[string]): bool =
75+ let lower = s.toLowerAscii
76+ for p in patterns:
77+ if p in lower: return true
78+ false
79+
80+iterator elements(n: XmlNode): XmlNode =
81+ ## Depth-first over element nodes, the root included.
82+ var stack = @[n]
83+ while stack.len > 0:
84+ let cur = stack.pop()
85+ if cur.kind == xnElement:
86+ yield cur
87+ for i in countdown(cur.len - 1, 0):
88+ stack.add cur[i]
89+
90+proc textLength(n: XmlNode): int =
91+ case n.kind
92+ of xnText, xnCData: result = n.text.strip().len
93+ of xnElement:
94+ for c in n: result += textLength(c)
95+ else: discard
96+
97+# --- finding the article ---------------------------------------------------
98+
99+proc findByTag(root: XmlNode, tag: string): Option[XmlNode] =
100+ for el in elements(root):
101+ if el.tag == tag: return some(el)
102+ none(XmlNode)
103+
104+proc findByAttr(root: XmlNode, key, value: string): Option[XmlNode] =
105+ for el in elements(root):
106+ if attrOf(el, key) == value: return some(el)
107+ none(XmlNode)
108+
109+proc findByContentClass(root: XmlNode): Option[XmlNode] =
110+ ## The widest net, so it runs last: pick the largest element whose class or
111+ ## id looks like a content wrapper. Size is the tie-breaker because these
112+ ## patterns nest -- "main-content" often wraps "entry-content".
113+ var best: XmlNode
114+ var bestLen = 0
115+ for el in elements(root):
116+ for key in ["class", "id"]:
117+ let v = attrOf(el, key)
118+ if v.len == 0 or not containsAny(v, ContentPatterns): continue
119+ let tl = textLength(el)
120+ if tl > bestLen and tl >= MinContentLength:
121+ best = el
122+ bestLen = tl
123+ if best == nil: none(XmlNode) else: some(best)
124+
125+proc findLargestTextNode(root: XmlNode): Option[XmlNode] =
126+ ## Last resort for pages with no semantic markup at all. Only div and
127+ ## section are considered: without that, the winner is always <body>.
128+ var best: XmlNode
129+ var bestLen = 0
130+ for el in elements(root):
131+ if el.tag notin ["div", "section"]: continue
132+ let tl = textLength(el)
133+ if tl > bestLen and tl >= MinContentLength:
134+ best = el
135+ bestLen = tl
136+ if best == nil: none(XmlNode) else: some(best)
137+
138+proc findArticle*(doc: XmlNode): Option[XmlNode] =
139+ for strategy in [
140+ proc(n: XmlNode): Option[XmlNode] = findByTag(n, "article"),
141+ proc(n: XmlNode): Option[XmlNode] = findByAttr(n, "role", "main"),
142+ proc(n: XmlNode): Option[XmlNode] = findByTag(n, "main"),
143+ proc(n: XmlNode): Option[XmlNode] = findByAttr(n, "itemprop", "articleBody"),
144+ findByContentClass,
145+ findLargestTextNode,
146+ ]:
147+ let found = strategy(doc)
148+ if found.isSome: return found
149+ none(XmlNode)
150+
151+# --- stripping furniture ---------------------------------------------------
152+
153+proc isUnwanted*(n: XmlNode): bool =
154+ if n.kind != xnElement: return false
155+ if n.tag in UnwantedTags: return true
156+ if containsAny(attrOf(n, "class"), UnwantedClassPatterns): return true
157+ if containsAny(attrOf(n, "id"), UnwantedIdPatterns): return true
158+ if attrOf(n, "role").toLowerAscii in UnwantedRoles: return true
159+ false
160+
161+proc removeUnwanted*(n: XmlNode) =
162+ ## Depth-first, back to front: deleting by index invalidates the indices
163+ ## after it, so walking forwards would skip a sibling after every removal.
164+ if n.kind != xnElement: return
165+ for i in countdown(n.len - 1, 0):
166+ if isUnwanted(n[i]):
167+ n.delete(i)
168+ else:
169+ removeUnwanted(n[i])
170+
171+# --- rendering -------------------------------------------------------------
172+
173+proc isHttpUrl(s: string): bool =
174+ s.startsWith("http://") or s.startsWith("https://")
175+
176+proc isReachableUrl(s: string): bool =
177+ isHttpUrl(s) or s.startsWith("//")
178+
179+proc resolveImgSrc(n: XmlNode): string =
180+ ## Lazy-loaded images put a placeholder in `src` and the real URL in a
181+ ## data attribute. Taking the first *reachable* one avoids rendering a
182+ ## 1x1 gif where the picture should be.
183+ for key in ["src", "data-src", "data-lazy-src"]:
184+ let v = attrOf(n, key)
185+ if v.len > 0 and isReachableUrl(v): return v
186+ ""
187+
188+proc escapeText(s: string): string =
189+ for c in s:
190+ case c
191+ of '<': result.add "&lt;"
192+ of '>': result.add "&gt;"
193+ of '&': result.add "&amp;"
194+ else: result.add c
195+
196+proc escapeAttr(s: string): string =
197+ escapeText(s).replace("\"", "&quot;")
198+
199+proc allowedFor(tag, key: string): bool =
200+ if tag == "img": key in ImgAllowedAttrs else: key in AllowedAttrs
201+
202+proc renderInto(buf: var string, n: XmlNode)
203+
204+proc renderChildren(buf: var string, n: XmlNode) =
205+ for c in n:
206+ renderInto(buf, c)
207+
208+proc renderInto(buf: var string, n: XmlNode) =
209+ case n.kind
210+ of xnText, xnCData:
211+ buf.add escapeText(n.text)
212+ return
213+ of xnElement: discard
214+ else:
215+ # Comments, entities and processing instructions carry nothing a reader
216+ # needs and are a needless way to smuggle markup through.
217+ return
218+
219+ let tag = n.tag
220+
221+ if tag == "a":
222+ let href = attrOf(n, "href")
223+ # A link that goes nowhere useful is rendered as its text, so the words
224+ # survive without an anchor pointing at the scraped site's internals.
225+ if not isReachableUrl(href):
226+ renderChildren(buf, n)
227+ return
228+ buf.add "<a href=\"" & escapeAttr(href) & "\" rel=\"noopener noreferrer\">"
229+ renderChildren(buf, n)
230+ buf.add "</a>"
231+ return
232+
233+ if tag == "img":
234+ let src = resolveImgSrc(n)
235+ if src.len == 0: return
236+ buf.add "<img src=\"" & escapeAttr(src) & "\""
237+ if n.attrs != nil:
238+ for key, value in n.attrs:
239+ if allowedFor(tag, key):
240+ buf.add " " & key & "=\"" & escapeAttr(value) & "\""
241+ buf.add ">"
242+ return
243+
244+ buf.add "<" & tag
245+ if n.attrs != nil:
246+ for key, value in n.attrs:
247+ if allowedFor(tag, key):
248+ buf.add " " & key & "=\"" & escapeAttr(value) & "\""
249+ buf.add ">"
250+
251+ if tag in VoidElements:
252+ return
253+ renderChildren(buf, n)
254+ buf.add "</" & tag & ">"
255+
256+proc renderNode*(n: XmlNode): string =
257+ renderChildren(result, n)
258+ result = result.strip()
259+
260+# --- entry point -----------------------------------------------------------
261+
262+proc extractArticle*(body: string): string =
263+ ## Pull the readable article out of a page.
264+ ##
265+ ## Raises rather than returning something thin: a caller that gets a nav
266+ ## menu back has no way to tell it apart from a short article, whereas an
267+ ## error lets it fall back to the feed's own summary.
268+ if body.strip().len == 0:
269+ raise newException(ScrapeError, "empty document")
270+
271+ let doc =
272+ try: parseHtml(body)
273+ except CatchableError as e:
274+ raise newException(ScrapeError, "could not parse HTML: " & e.msg)
275+
276+ let article = findArticle(doc)
277+ if article.isNone:
278+ raise newException(ScrapeError, "no article content found")
279+
280+ let node = article.get
281+ removeUnwanted(node)
282+ result = renderNode(node)
283+
284+ if textLength(node) < MinContentLength:
285+ raise newException(ScrapeError, "extracted content too short")
new file mode 100644
@@ -0,0 +1,285 @@
1+## Extracting an article from a page, ported from internal/scraper/scraper.go.
2+##
3+## Feeds routinely ship a first paragraph and a "read more" link, so the
4+## reader view needs the page itself. This is a readability-style extractor:
5+## find the node most likely to be the article, strip the furniture around
6+## it, and re-render a small whitelisted subset of HTML.
7+##
8+## Two things are deliberate:
9+##
10+## * The output is sanitised HTML, not text. Paragraphs, headings, lists,
11+## code blocks and images are what make a long article readable, and
12+## flattening them to text loses more than the markup costs.
13+## * Nothing is trusted. Attributes are whitelisted rather than blacklisted,
14+## and `script`, `style`, `iframe` and friends never survive, because this
15+## renders HTML fetched from an arbitrary site into the reader's client.
16+
17+import std/[htmlparser, options, strtabs, strutils, xmltree]
18+
19+type
20+ ScrapeError* = object of CatchableError
21+
22+const
23+ MinContentLength* = 200
24+ ## Below this a candidate is furniture, not an article.
25+
26+ ## Ordered: the first that matches wins. Semantic markup beats guessing,
27+ ## so <article> and role=main come before class-name heuristics.
28+ ContentPatterns = [
29+ "post-content", "entry-content", "article-content",
30+ "article-body", "article__body", "article__content",
31+ "story-body", "story-content", "content-body",
32+ "post-body", "post-entry", "entry-body",
33+ "blog-content", "blog-post", "wp-content",
34+ "main-content", "page-content", "body-content",
35+ ]
36+
37+ UnwantedTags = [
38+ "script", "style", "nav", "header", "footer", "aside", "noscript",
39+ "iframe", "form", "svg", "button", "input", "textarea", "select",
40+ ]
41+
42+ UnwantedRoles = ["complementary", "banner", "contentinfo", "navigation"]
43+
44+ UnwantedClassPatterns = [
45+ "comment", "sidebar", "advertisement", "ad-banner",
46+ "social-share", "share-button", "newsletter", "popup",
47+ "cookie", "paywall", "related-post", "related-article",
48+ "taboola", "outbrain", "disqus",
49+ ]
50+
51+ ## Narrower than the class list: an id of "header" is reliably the page
52+ ## header, while a *class* of "header" is often the article's own.
53+ UnwantedIdPatterns = ["comment", "sidebar", "footer", "header", "nav", "disqus"]
54+
55+ VoidElements = [
56+ "br", "hr", "img", "input", "meta", "link", "area", "base", "col",
57+ "embed", "source", "track", "wbr",
58+ ]
59+
60+ AllowedAttrs = [
61+ "href", "src", "alt", "title", "class", "id", "width", "height",
62+ "type", "controls", "preload", "poster", "cite", "datetime",
63+ "colspan", "rowspan", "loading", "decoding", "itemprop",
64+ "role", "aria-label", "aria-hidden",
65+ ]
66+
67+ ## Images keep only presentational attributes; `src` is resolved separately
68+ ## so that lazy-loading placeholders do not survive as broken images.
69+ ImgAllowedAttrs = ["alt", "width", "height", "class"]
70+
71+proc attrOf(n: XmlNode, key: string): string =
72+ if n.kind != xnElement or n.attrs == nil: "" else: n.attrs.getOrDefault(key)
73+
74+proc containsAny(s: string, patterns: openArray[string]): bool =
75+ let lower = s.toLowerAscii
76+ for p in patterns:
77+ if p in lower: return true
78+ false
79+
80+iterator elements(n: XmlNode): XmlNode =
81+ ## Depth-first over element nodes, the root included.
82+ var stack = @[n]
83+ while stack.len > 0:
84+ let cur = stack.pop()
85+ if cur.kind == xnElement:
86+ yield cur
87+ for i in countdown(cur.len - 1, 0):
88+ stack.add cur[i]
89+
90+proc textLength(n: XmlNode): int =
91+ case n.kind
92+ of xnText, xnCData: result = n.text.strip().len
93+ of xnElement:
94+ for c in n: result += textLength(c)
95+ else: discard
96+
97+# --- finding the article ---------------------------------------------------
98+
99+proc findByTag(root: XmlNode, tag: string): Option[XmlNode] =
100+ for el in elements(root):
101+ if el.tag == tag: return some(el)
102+ none(XmlNode)
103+
104+proc findByAttr(root: XmlNode, key, value: string): Option[XmlNode] =
105+ for el in elements(root):
106+ if attrOf(el, key) == value: return some(el)
107+ none(XmlNode)
108+
109+proc findByContentClass(root: XmlNode): Option[XmlNode] =
110+ ## The widest net, so it runs last: pick the largest element whose class or
111+ ## id looks like a content wrapper. Size is the tie-breaker because these
112+ ## patterns nest -- "main-content" often wraps "entry-content".
113+ var best: XmlNode
114+ var bestLen = 0
115+ for el in elements(root):
116+ for key in ["class", "id"]:
117+ let v = attrOf(el, key)
118+ if v.len == 0 or not containsAny(v, ContentPatterns): continue
119+ let tl = textLength(el)
120+ if tl > bestLen and tl >= MinContentLength:
121+ best = el
122+ bestLen = tl
123+ if best == nil: none(XmlNode) else: some(best)
124+
125+proc findLargestTextNode(root: XmlNode): Option[XmlNode] =
126+ ## Last resort for pages with no semantic markup at all. Only div and
127+ ## section are considered: without that, the winner is always <body>.
128+ var best: XmlNode
129+ var bestLen = 0
130+ for el in elements(root):
131+ if el.tag notin ["div", "section"]: continue
132+ let tl = textLength(el)
133+ if tl > bestLen and tl >= MinContentLength:
134+ best = el
135+ bestLen = tl
136+ if best == nil: none(XmlNode) else: some(best)
137+
138+proc findArticle*(doc: XmlNode): Option[XmlNode] =
139+ for strategy in [
140+ proc(n: XmlNode): Option[XmlNode] = findByTag(n, "article"),
141+ proc(n: XmlNode): Option[XmlNode] = findByAttr(n, "role", "main"),
142+ proc(n: XmlNode): Option[XmlNode] = findByTag(n, "main"),
143+ proc(n: XmlNode): Option[XmlNode] = findByAttr(n, "itemprop", "articleBody"),
144+ findByContentClass,
145+ findLargestTextNode,
146+ ]:
147+ let found = strategy(doc)
148+ if found.isSome: return found
149+ none(XmlNode)
150+
151+# --- stripping furniture ---------------------------------------------------
152+
153+proc isUnwanted*(n: XmlNode): bool =
154+ if n.kind != xnElement: return false
155+ if n.tag in UnwantedTags: return true
156+ if containsAny(attrOf(n, "class"), UnwantedClassPatterns): return true
157+ if containsAny(attrOf(n, "id"), UnwantedIdPatterns): return true
158+ if attrOf(n, "role").toLowerAscii in UnwantedRoles: return true
159+ false
160+
161+proc removeUnwanted*(n: XmlNode) =
162+ ## Depth-first, back to front: deleting by index invalidates the indices
163+ ## after it, so walking forwards would skip a sibling after every removal.
164+ if n.kind != xnElement: return
165+ for i in countdown(n.len - 1, 0):
166+ if isUnwanted(n[i]):
167+ n.delete(i)
168+ else:
169+ removeUnwanted(n[i])
170+
171+# --- rendering -------------------------------------------------------------
172+
173+proc isHttpUrl(s: string): bool =
174+ s.startsWith("http://") or s.startsWith("https://")
175+
176+proc isReachableUrl(s: string): bool =
177+ isHttpUrl(s) or s.startsWith("//")
178+
179+proc resolveImgSrc(n: XmlNode): string =
180+ ## Lazy-loaded images put a placeholder in `src` and the real URL in a
181+ ## data attribute. Taking the first *reachable* one avoids rendering a
182+ ## 1x1 gif where the picture should be.
183+ for key in ["src", "data-src", "data-lazy-src"]:
184+ let v = attrOf(n, key)
185+ if v.len > 0 and isReachableUrl(v): return v
186+ ""
187+
188+proc escapeText(s: string): string =
189+ for c in s:
190+ case c
191+ of '<': result.add "&lt;"
192+ of '>': result.add "&gt;"
193+ of '&': result.add "&amp;"
194+ else: result.add c
195+
196+proc escapeAttr(s: string): string =
197+ escapeText(s).replace("\"", "&quot;")
198+
199+proc allowedFor(tag, key: string): bool =
200+ if tag == "img": key in ImgAllowedAttrs else: key in AllowedAttrs
201+
202+proc renderInto(buf: var string, n: XmlNode)
203+
204+proc renderChildren(buf: var string, n: XmlNode) =
205+ for c in n:
206+ renderInto(buf, c)
207+
208+proc renderInto(buf: var string, n: XmlNode) =
209+ case n.kind
210+ of xnText, xnCData:
211+ buf.add escapeText(n.text)
212+ return
213+ of xnElement: discard
214+ else:
215+ # Comments, entities and processing instructions carry nothing a reader
216+ # needs and are a needless way to smuggle markup through.
217+ return
218+
219+ let tag = n.tag
220+
221+ if tag == "a":
222+ let href = attrOf(n, "href")
223+ # A link that goes nowhere useful is rendered as its text, so the words
224+ # survive without an anchor pointing at the scraped site's internals.
225+ if not isReachableUrl(href):
226+ renderChildren(buf, n)
227+ return
228+ buf.add "<a href=\"" & escapeAttr(href) & "\" rel=\"noopener noreferrer\">"
229+ renderChildren(buf, n)
230+ buf.add "</a>"
231+ return
232+
233+ if tag == "img":
234+ let src = resolveImgSrc(n)
235+ if src.len == 0: return
236+ buf.add "<img src=\"" & escapeAttr(src) & "\""
237+ if n.attrs != nil:
238+ for key, value in n.attrs:
239+ if allowedFor(tag, key):
240+ buf.add " " & key & "=\"" & escapeAttr(value) & "\""
241+ buf.add ">"
242+ return
243+
244+ buf.add "<" & tag
245+ if n.attrs != nil:
246+ for key, value in n.attrs:
247+ if allowedFor(tag, key):
248+ buf.add " " & key & "=\"" & escapeAttr(value) & "\""
249+ buf.add ">"
250+
251+ if tag in VoidElements:
252+ return
253+ renderChildren(buf, n)
254+ buf.add "</" & tag & ">"
255+
256+proc renderNode*(n: XmlNode): string =
257+ renderChildren(result, n)
258+ result = result.strip()
259+
260+# --- entry point -----------------------------------------------------------
261+
262+proc extractArticle*(body: string): string =
263+ ## Pull the readable article out of a page.
264+ ##
265+ ## Raises rather than returning something thin: a caller that gets a nav
266+ ## menu back has no way to tell it apart from a short article, whereas an
267+ ## error lets it fall back to the feed's own summary.
268+ if body.strip().len == 0:
269+ raise newException(ScrapeError, "empty document")
270+
271+ let doc =
272+ try: parseHtml(body)
273+ except CatchableError as e:
274+ raise newException(ScrapeError, "could not parse HTML: " & e.msg)
275+
276+ let article = findArticle(doc)
277+ if article.isNone:
278+ raise newException(ScrapeError, "no article content found")
279+
280+ let node = article.get
281+ removeUnwanted(node)
282+ result = renderNode(node)
283+
284+ if textLength(node) < MinContentLength:
285+ raise newException(ScrapeError, "extracted content too short")
added spike/nim/scraper_probe.nim +224 -0
new file mode 100644
@@ -0,0 +1,224 @@
1+## Checks the article extractor.
2+##
3+## nim c -r -d:ssl scraper_probe.nim
4+##
5+## Two things are being tested, and the second matters more. Extraction: does
6+## the right node win, and does the furniture go. Sanitisation: this renders
7+## HTML from an arbitrary site into a reader's client, so anything that could
8+## carry script must not survive -- and that has to be asserted, because a
9+## page that renders correctly proves nothing about what was stripped.
10+
11+import std/[htmlparser, httpclient, options, strformat, strutils, xmltree]
12+import atproto/[feedparser, scraper]
13+
14+var failures = 0
15+
16+proc report(name: string, ok: bool, detail = "") =
17+ if not ok: inc failures
18+ let label = if ok: "PASS" else: "FAIL"
19+ if detail.len > 0: echo &" {label} {name} -- {detail}"
20+ else: echo &" {label} {name}"
21+
22+# Long enough to clear MinContentLength.
23+const body = """<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit,
24+sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad
25+minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
26+commodo consequat. Duis aute irure dolor in reprehenderit.</p>"""
27+
28+proc page(inner: string): string =
29+ &"""<!DOCTYPE html><html><head><title>t</title></head><body>{inner}</body></html>"""
30+
31+proc main() =
32+ echo "Scraper probe"
33+ echo ""
34+
35+ # --- which node wins ----------------------------------------------------
36+ block:
37+ let html = page(&"""
38+ <nav>menu menu menu</nav>
39+ <article><h1>The headline</h1>{body}</article>
40+ <footer>footer text</footer>""")
41+ let got = extractArticle(html)
42+ report("<article> is preferred", "The headline" in got)
43+ report("nav is stripped", "menu menu" notin got)
44+ report("footer is stripped", "footer text" notin got)
45+
46+ block:
47+ # Semantic markup should beat a class-name guess.
48+ let html = page(&"""
49+ <div class="post-content">wrong one, but long enough {body}</div>
50+ <main><h1>Main wins</h1>{body}</main>""")
51+ let got = extractArticle(html)
52+ report("role/main beats a content class", "Main wins" in got)
53+ report("the losing candidate is not included", "wrong one" notin got)
54+
55+ block:
56+ let html = page(&"""<div role="main"><h1>By role</h1>{body}</div>""")
57+ report("role=main is found", "By role" in extractArticle(html))
58+
59+ block:
60+ let html = page(&"""<div itemprop="articleBody"><h1>By itemprop</h1>{body}</div>""")
61+ report("itemprop=articleBody is found",
62+ "By itemprop" in extractArticle(html))
63+
64+ block:
65+ let html = page(&"""<div class="entry-content"><h1>By class</h1>{body}</div>""")
66+ report("a content class is found", "By class" in extractArticle(html))
67+
68+ block:
69+ # These patterns nest; the larger should win.
70+ let html = page(&"""
71+ <div class="main-content"><div class="entry-content">
72+ <h1>Inner</h1>{body}{body}
73+ </div></div>""")
74+ let got = extractArticle(html)
75+ report("the largest matching container wins", "Inner" in got)
76+
77+ block:
78+ # No semantic markup at all.
79+ let html = page(&"""<div><h1>Largest</h1>{body}</div><div>tiny</div>""")
80+ report("it falls back to the largest text block",
81+ "Largest" in extractArticle(html))
82+
83+ # --- furniture ----------------------------------------------------------
84+ block:
85+ let html = page(&"""<article>{body}
86+ <div class="related-articles">related junk</div>
87+ <div id="disqus_thread">comments here</div>
88+ <aside>aside junk</aside>
89+ <div role="complementary">complementary junk</div>
90+ <section class="newsletter">subscribe now</section>
91+ </article>""")
92+ let got = extractArticle(html)
93+ for junk in ["related junk", "comments here", "aside junk",
94+ "complementary junk", "subscribe now"]:
95+ report(&"stripped: {junk}", junk notin got)
96+
97+ block:
98+ # Removal is back-to-front for a reason: deleting by index shifts the
99+ # siblings after it, so a forward walk skips one after each removal.
100+ let html = page(&"""<article>
101+ <nav>one</nav><nav>two</nav><nav>three</nav><nav>four</nav>{body}
102+ </article>""")
103+ let got = extractArticle(html)
104+ report("consecutive removals do not skip siblings",
105+ "one" notin got and "two" notin got and
106+ "three" notin got and "four" notin got)
107+
108+ # --- sanitisation -------------------------------------------------------
109+ block:
110+ let html = page(&"""<article>
111+ <script>alert('xss')</script>
112+ <style>body {{ color: red }}</style>
113+ <iframe src="https://evil.test"></iframe>
114+ <p onclick="steal()" data-tracking="123">clean paragraph</p>
115+ {body}
116+ </article>""")
117+ let got = extractArticle(html)
118+ report("script content never survives",
119+ "alert" notin got and "<script" notin got)
120+ report("style content never survives", "color: red" notin got)
121+ report("iframes never survive", "iframe" notin got and "evil.test" notin got)
122+ # Whitelisted, not blacklisted: an unknown attribute is dropped by
123+ # default rather than needing to be anticipated.
124+ report("event handlers are dropped", "onclick" notin got)
125+ report("unknown data attributes are dropped", "data-tracking" notin got)
126+ report("the text itself is kept", "clean paragraph" in got)
127+
128+ block:
129+ let html = page(&"""<article>
130+ <a href="javascript:alert(1)">bad link</a>
131+ <a href="/relative">relative link</a>
132+ <a href="https://ok.test/x">good link</a>
133+ {body}</article>""")
134+ let got = extractArticle(html)
135+ report("javascript: URLs are not rendered as links",
136+ "javascript:" notin got)
137+ report("but their text survives", "bad link" in got)
138+ report("relative links are flattened to text",
139+ "relative link" in got and "\"/relative\"" notin got)
140+ report("absolute links are kept", "https://ok.test/x" in got)
141+ # An anchor into a scraped page should not hand the target window a
142+ # reference back.
143+ report("links carry rel=noopener", "noopener" in got)
144+
145+ block:
146+ let html = page(&"""<article>
147+ <img src="data:image/gif;base64,R0lGOD" data-src="https://cdn.test/real.jpg" alt="real">
148+ <img src="https://cdn.test/plain.jpg" alt="plain">
149+ <img alt="no source at all">
150+ {body}</article>""")
151+ let got = extractArticle(html)
152+ # A placeholder rendered as-is is a broken image where the picture goes.
153+ report("a lazy-loaded image resolves to its real source",
154+ "cdn.test/real.jpg" in got and "data:image" notin got)
155+ report("a plain image is kept", "cdn.test/plain.jpg" in got)
156+ report("an image with no usable source is dropped",
157+ "no source at all" notin got)
158+ report("alt text is kept", "alt=\"real\"" in got)
159+
160+ block:
161+ let html = page(&"""<article><p>a &lt;b&gt; &amp; c</p>{body}</article>""")
162+ let got = extractArticle(html)
163+ report("text is re-escaped, not passed through",
164+ "&lt;b&gt;" in got and "<b>" notin got, "")
165+
166+ # --- structure is preserved ---------------------------------------------
167+ block:
168+ let html = page(&"""<article>
169+ <h2>A heading</h2>
170+ <ul><li>first</li><li>second</li></ul>
171+ <pre><code>let x = 1</code></pre>
172+ <blockquote>quoted</blockquote>
173+ {body}</article>""")
174+ let got = extractArticle(html)
175+ for tag in ["<h2>", "<ul>", "<li>", "<pre>", "<code>", "<blockquote>"]:
176+ report(&"structure kept: {tag}", tag in got)
177+
178+ # --- refusal ------------------------------------------------------------
179+ block:
180+ proc refuses(html: string): bool =
181+ try:
182+ discard extractArticle(html)
183+ false
184+ except ScrapeError:
185+ true
186+ report("an empty document is refused", refuses(" "))
187+ # Returning a nav menu would be indistinguishable to the caller from a
188+ # short article; an error lets it fall back to the feed summary.
189+ report("a page with no article is refused",
190+ refuses(page("<nav>just a menu</nav>")))
191+ report("a too-short article is refused",
192+ refuses(page("<article><p>tiny</p></article>")))
193+
194+ # --- a real page --------------------------------------------------------
195+ # The URL is taken from the feed rather than hard-coded, so this cannot rot
196+ # when a blog reorganises its permalinks.
197+ block:
198+ try:
199+ let client = newHttpClient(timeout = 20_000,
200+ userAgent = "glean-nim-spike/0.1")
201+ defer: client.close()
202+ let feed = parseFeed(client.getContent("https://blog.rust-lang.org/feed.xml"),
203+ "https://blog.rust-lang.org/feed.xml")
204+ if feed.articles.len == 0 or feed.articles[0].url.len == 0:
205+ echo " SKIP a real article page -- feed had no article URLs"
206+ else:
207+ let url = feed.articles[0].url
208+ let got = extractArticle(client.getContent(url))
209+ report("a real article page extracts", got.len > 1000,
210+ &"{got.len} chars from {url.split('/')[^2]}")
211+ report("and carries no script", "<script" notin got)
212+ report("and keeps its paragraphs", "<p>" in got)
213+ except CatchableError as e:
214+ echo &" SKIP a real article page -- {e.msg}"
215+
216+ echo ""
217+ if failures == 0:
218+ echo "RESULT: the scraper extracts and sanitises."
219+ else:
220+ echo &"RESULT: {failures} check(s) failed."
221+ quit 1
222+
223+when isMainModule:
224+ main()
new file mode 100644
@@ -0,0 +1,224 @@
1+## Checks the article extractor.
2+##
3+## nim c -r -d:ssl scraper_probe.nim
4+##
5+## Two things are being tested, and the second matters more. Extraction: does
6+## the right node win, and does the furniture go. Sanitisation: this renders
7+## HTML from an arbitrary site into a reader's client, so anything that could
8+## carry script must not survive -- and that has to be asserted, because a
9+## page that renders correctly proves nothing about what was stripped.
10+
11+import std/[htmlparser, httpclient, options, strformat, strutils, xmltree]
12+import atproto/[feedparser, scraper]
13+
14+var failures = 0
15+
16+proc report(name: string, ok: bool, detail = "") =
17+ if not ok: inc failures
18+ let label = if ok: "PASS" else: "FAIL"
19+ if detail.len > 0: echo &" {label} {name} -- {detail}"
20+ else: echo &" {label} {name}"
21+
22+# Long enough to clear MinContentLength.
23+const body = """<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit,
24+sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad
25+minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
26+commodo consequat. Duis aute irure dolor in reprehenderit.</p>"""
27+
28+proc page(inner: string): string =
29+ &"""<!DOCTYPE html><html><head><title>t</title></head><body>{inner}</body></html>"""
30+
31+proc main() =
32+ echo "Scraper probe"
33+ echo ""
34+
35+ # --- which node wins ----------------------------------------------------
36+ block:
37+ let html = page(&"""
38+ <nav>menu menu menu</nav>
39+ <article><h1>The headline</h1>{body}</article>
40+ <footer>footer text</footer>""")
41+ let got = extractArticle(html)
42+ report("<article> is preferred", "The headline" in got)
43+ report("nav is stripped", "menu menu" notin got)
44+ report("footer is stripped", "footer text" notin got)
45+
46+ block:
47+ # Semantic markup should beat a class-name guess.
48+ let html = page(&"""
49+ <div class="post-content">wrong one, but long enough {body}</div>
50+ <main><h1>Main wins</h1>{body}</main>""")
51+ let got = extractArticle(html)
52+ report("role/main beats a content class", "Main wins" in got)
53+ report("the losing candidate is not included", "wrong one" notin got)
54+
55+ block:
56+ let html = page(&"""<div role="main"><h1>By role</h1>{body}</div>""")
57+ report("role=main is found", "By role" in extractArticle(html))
58+
59+ block:
60+ let html = page(&"""<div itemprop="articleBody"><h1>By itemprop</h1>{body}</div>""")
61+ report("itemprop=articleBody is found",
62+ "By itemprop" in extractArticle(html))
63+
64+ block:
65+ let html = page(&"""<div class="entry-content"><h1>By class</h1>{body}</div>""")
66+ report("a content class is found", "By class" in extractArticle(html))
67+
68+ block:
69+ # These patterns nest; the larger should win.
70+ let html = page(&"""
71+ <div class="main-content"><div class="entry-content">
72+ <h1>Inner</h1>{body}{body}
73+ </div></div>""")
74+ let got = extractArticle(html)
75+ report("the largest matching container wins", "Inner" in got)
76+
77+ block:
78+ # No semantic markup at all.
79+ let html = page(&"""<div><h1>Largest</h1>{body}</div><div>tiny</div>""")
80+ report("it falls back to the largest text block",
81+ "Largest" in extractArticle(html))
82+
83+ # --- furniture ----------------------------------------------------------
84+ block:
85+ let html = page(&"""<article>{body}
86+ <div class="related-articles">related junk</div>
87+ <div id="disqus_thread">comments here</div>
88+ <aside>aside junk</aside>
89+ <div role="complementary">complementary junk</div>
90+ <section class="newsletter">subscribe now</section>
91+ </article>""")
92+ let got = extractArticle(html)
93+ for junk in ["related junk", "comments here", "aside junk",
94+ "complementary junk", "subscribe now"]:
95+ report(&"stripped: {junk}", junk notin got)
96+
97+ block:
98+ # Removal is back-to-front for a reason: deleting by index shifts the
99+ # siblings after it, so a forward walk skips one after each removal.
100+ let html = page(&"""<article>
101+ <nav>one</nav><nav>two</nav><nav>three</nav><nav>four</nav>{body}
102+ </article>""")
103+ let got = extractArticle(html)
104+ report("consecutive removals do not skip siblings",
105+ "one" notin got and "two" notin got and
106+ "three" notin got and "four" notin got)
107+
108+ # --- sanitisation -------------------------------------------------------
109+ block:
110+ let html = page(&"""<article>
111+ <script>alert('xss')</script>
112+ <style>body {{ color: red }}</style>
113+ <iframe src="https://evil.test"></iframe>
114+ <p onclick="steal()" data-tracking="123">clean paragraph</p>
115+ {body}
116+ </article>""")
117+ let got = extractArticle(html)
118+ report("script content never survives",
119+ "alert" notin got and "<script" notin got)
120+ report("style content never survives", "color: red" notin got)
121+ report("iframes never survive", "iframe" notin got and "evil.test" notin got)
122+ # Whitelisted, not blacklisted: an unknown attribute is dropped by
123+ # default rather than needing to be anticipated.
124+ report("event handlers are dropped", "onclick" notin got)
125+ report("unknown data attributes are dropped", "data-tracking" notin got)
126+ report("the text itself is kept", "clean paragraph" in got)
127+
128+ block:
129+ let html = page(&"""<article>
130+ <a href="javascript:alert(1)">bad link</a>
131+ <a href="/relative">relative link</a>
132+ <a href="https://ok.test/x">good link</a>
133+ {body}</article>""")
134+ let got = extractArticle(html)
135+ report("javascript: URLs are not rendered as links",
136+ "javascript:" notin got)
137+ report("but their text survives", "bad link" in got)
138+ report("relative links are flattened to text",
139+ "relative link" in got and "\"/relative\"" notin got)
140+ report("absolute links are kept", "https://ok.test/x" in got)
141+ # An anchor into a scraped page should not hand the target window a
142+ # reference back.
143+ report("links carry rel=noopener", "noopener" in got)
144+
145+ block:
146+ let html = page(&"""<article>
147+ <img src="data:image/gif;base64,R0lGOD" data-src="https://cdn.test/real.jpg" alt="real">
148+ <img src="https://cdn.test/plain.jpg" alt="plain">
149+ <img alt="no source at all">
150+ {body}</article>""")
151+ let got = extractArticle(html)
152+ # A placeholder rendered as-is is a broken image where the picture goes.
153+ report("a lazy-loaded image resolves to its real source",
154+ "cdn.test/real.jpg" in got and "data:image" notin got)
155+ report("a plain image is kept", "cdn.test/plain.jpg" in got)
156+ report("an image with no usable source is dropped",
157+ "no source at all" notin got)
158+ report("alt text is kept", "alt=\"real\"" in got)
159+
160+ block:
161+ let html = page(&"""<article><p>a &lt;b&gt; &amp; c</p>{body}</article>""")
162+ let got = extractArticle(html)
163+ report("text is re-escaped, not passed through",
164+ "&lt;b&gt;" in got and "<b>" notin got, "")
165+
166+ # --- structure is preserved ---------------------------------------------
167+ block:
168+ let html = page(&"""<article>
169+ <h2>A heading</h2>
170+ <ul><li>first</li><li>second</li></ul>
171+ <pre><code>let x = 1</code></pre>
172+ <blockquote>quoted</blockquote>
173+ {body}</article>""")
174+ let got = extractArticle(html)
175+ for tag in ["<h2>", "<ul>", "<li>", "<pre>", "<code>", "<blockquote>"]:
176+ report(&"structure kept: {tag}", tag in got)
177+
178+ # --- refusal ------------------------------------------------------------
179+ block:
180+ proc refuses(html: string): bool =
181+ try:
182+ discard extractArticle(html)
183+ false
184+ except ScrapeError:
185+ true
186+ report("an empty document is refused", refuses(" "))
187+ # Returning a nav menu would be indistinguishable to the caller from a
188+ # short article; an error lets it fall back to the feed summary.
189+ report("a page with no article is refused",
190+ refuses(page("<nav>just a menu</nav>")))
191+ report("a too-short article is refused",
192+ refuses(page("<article><p>tiny</p></article>")))
193+
194+ # --- a real page --------------------------------------------------------
195+ # The URL is taken from the feed rather than hard-coded, so this cannot rot
196+ # when a blog reorganises its permalinks.
197+ block:
198+ try:
199+ let client = newHttpClient(timeout = 20_000,
200+ userAgent = "glean-nim-spike/0.1")
201+ defer: client.close()
202+ let feed = parseFeed(client.getContent("https://blog.rust-lang.org/feed.xml"),
203+ "https://blog.rust-lang.org/feed.xml")
204+ if feed.articles.len == 0 or feed.articles[0].url.len == 0:
205+ echo " SKIP a real article page -- feed had no article URLs"
206+ else:
207+ let url = feed.articles[0].url
208+ let got = extractArticle(client.getContent(url))
209+ report("a real article page extracts", got.len > 1000,
210+ &"{got.len} chars from {url.split('/')[^2]}")
211+ report("and carries no script", "<script" notin got)
212+ report("and keeps its paragraphs", "<p>" in got)
213+ except CatchableError as e:
214+ echo &" SKIP a real article page -- {e.msg}"
215+
216+ echo ""
217+ if failures == 0:
218+ echo "RESULT: the scraper extracts and sanitises."
219+ else:
220+ echo &"RESULT: {failures} check(s) failed."
221+ quit 1
222+
223+when isMainModule:
224+ main()