rickub
nandi/gleanpublic⑂ Fork 0
⑂ dc0dcf0
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Add support for parsing RSS1 (RDF) feedsUnverified

Julien Robert committed 2026-04-21T14:31:37+02:00 Browse files
dc0dcf0 parent: a3d0d15
modified internal/feed/parser.go +59 -0
@@ -80,6 +80,24 @@ type atomFeed struct {
8080 } `xml:"entry"`
8181 }
8282
83+type rdfFeed struct {
84+ XMLName xml.Name `xml:"http://www.w3.org/1999/02/22-rdf-syntax-ns# RDF"`
85+ Channel struct {
86+ Title string `xml:"title"`
87+ Link string `xml:"link"`
88+ Description string `xml:"description"`
89+ } `xml:"channel"`
90+ Items []struct {
91+ About string `xml:"http://www.w3.org/1999/02/22-rdf-syntax-ns# about,attr"`
92+ Title string `xml:"title"`
93+ Link string `xml:"link"`
94+ Description string `xml:"description"`
95+ Content string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"`
96+ Creator string `xml:"http://purl.org/dc/elements/1.1/ creator"`
97+ Date string `xml:"http://purl.org/dc/elements/1.1/ date"`
98+ } `xml:"item"`
99+}
100+
83101 type jsonFeed struct {
84102 Version string `json:"version"`
85103 Title string `json:"title"`
@@ -179,6 +197,13 @@ func parseXMLFeed(data []byte, feedURL string) (*ParseResult, error) {
179197 }
180198 }
181199
200+ var rdf rdfFeed
201+ if err := makeXMLDecoder(data).Decode(&rdf); err == nil {
202+ if rdf.XMLName.Local == "RDF" {
203+ return convertRDF(&rdf, feedURL), nil
204+ }
205+ }
206+
182207 return nil, fmt.Errorf("unable to detect feed format")
183208 }
184209
@@ -212,6 +237,40 @@ func convertRSS(rss *rssFeed, feedURL string) *ParseResult {
212237 return result
213238 }
214239
240+func convertRDF(rdf *rdfFeed, feedURL string) *ParseResult {
241+ result := &ParseResult{
242+ Feed: Feed{
243+ URL: feedURL,
244+ Title: rdf.Channel.Title,
245+ SiteURL: rdf.Channel.Link,
246+ Description: rdf.Channel.Description,
247+ Type: "rdf",
248+ },
249+ }
250+
251+ for _, item := range rdf.Items {
252+ guid := item.About
253+ if guid == "" {
254+ guid = item.Link
255+ }
256+ article := Article{
257+ GUID: guid,
258+ Title: item.Title,
259+ URL: item.Link,
260+ Content: item.Content,
261+ Summary: item.Description,
262+ Author: item.Creator,
263+ Published: parseTime(item.Date),
264+ }
265+ if article.GUID == "" {
266+ article.GUID = article.URL
267+ }
268+ result.Articles = append(result.Articles, article)
269+ }
270+
271+ return result
272+}
273+
215274 func convertAtom(atom *atomFeed, feedURL string) *ParseResult {
216275 result := &ParseResult{
217276 Feed: Feed{
@@ -80,6 +80,24 @@ type atomFeed struct {
80 } `xml:"entry"`80 } `xml:"entry"`
81 }81 }
82 82
83+type rdfFeed struct {
84+ XMLName xml.Name `xml:"http://www.w3.org/1999/02/22-rdf-syntax-ns# RDF"`
85+ Channel struct {
86+ Title string `xml:"title"`
87+ Link string `xml:"link"`
88+ Description string `xml:"description"`
89+ } `xml:"channel"`
90+ Items []struct {
91+ About string `xml:"http://www.w3.org/1999/02/22-rdf-syntax-ns# about,attr"`
92+ Title string `xml:"title"`
93+ Link string `xml:"link"`
94+ Description string `xml:"description"`
95+ Content string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"`
96+ Creator string `xml:"http://purl.org/dc/elements/1.1/ creator"`
97+ Date string `xml:"http://purl.org/dc/elements/1.1/ date"`
98+ } `xml:"item"`
99+}
100+
83 type jsonFeed struct {101 type jsonFeed struct {
84 Version string `json:"version"`102 Version string `json:"version"`
85 Title string `json:"title"`103 Title string `json:"title"`
@@ -179,6 +197,13 @@ func parseXMLFeed(data []byte, feedURL string) (*ParseResult, error) {
179 }197 }
180 }198 }
181 199
200+ var rdf rdfFeed
201+ if err := makeXMLDecoder(data).Decode(&rdf); err == nil {
202+ if rdf.XMLName.Local == "RDF" {
203+ return convertRDF(&rdf, feedURL), nil
204+ }
205+ }
206+
182 return nil, fmt.Errorf("unable to detect feed format")207 return nil, fmt.Errorf("unable to detect feed format")
183 }208 }
184 209
@@ -212,6 +237,40 @@ func convertRSS(rss *rssFeed, feedURL string) *ParseResult {
212 return result237 return result
213 }238 }
214 239
240+func convertRDF(rdf *rdfFeed, feedURL string) *ParseResult {
241+ result := &ParseResult{
242+ Feed: Feed{
243+ URL: feedURL,
244+ Title: rdf.Channel.Title,
245+ SiteURL: rdf.Channel.Link,
246+ Description: rdf.Channel.Description,
247+ Type: "rdf",
248+ },
249+ }
250+
251+ for _, item := range rdf.Items {
252+ guid := item.About
253+ if guid == "" {
254+ guid = item.Link
255+ }
256+ article := Article{
257+ GUID: guid,
258+ Title: item.Title,
259+ URL: item.Link,
260+ Content: item.Content,
261+ Summary: item.Description,
262+ Author: item.Creator,
263+ Published: parseTime(item.Date),
264+ }
265+ if article.GUID == "" {
266+ article.GUID = article.URL
267+ }
268+ result.Articles = append(result.Articles, article)
269+ }
270+
271+ return result
272+}
273+
215 func convertAtom(atom *atomFeed, feedURL string) *ParseResult {274 func convertAtom(atom *atomFeed, feedURL string) *ParseResult {
216 result := &ParseResult{275 result := &ParseResult{
217 Feed: Feed{276 Feed: Feed{
added internal/feed/parser_test.go +172 -0
new file mode 100644
@@ -0,0 +1,172 @@
1+package feed
2+
3+import (
4+ "strings"
5+ "testing"
6+ "time"
7+
8+ "gotest.tools/v3/assert"
9+)
10+
11+func TestParse_RSS1(t *testing.T) {
12+ rdf := `<?xml version="1.0" encoding="UTF-8"?>
13+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:dc="http://purl.org/dc/elements/1.1/">
14+ <channel>
15+ <title>Test Feed</title>
16+ <link>https://example.com</link>
17+ <description>A test feed</description>
18+ </channel>
19+ <item rdf:about="https://example.com/1">
20+ <title>First Item</title>
21+ <link>https://example.com/1</link>
22+ <dc:date>2026-04-21T10:00:00+09:00</dc:date>
23+ <dc:creator>Alice</dc:creator>
24+ <description>Description of first item</description>
25+ </item>
26+ <item rdf:about="https://example.com/2">
27+ <title>Second Item</title>
28+ <link>https://example.com/2</link>
29+ <dc:date>2026-04-20T08:00:00Z</dc:date>
30+ <description>Description of second item</description>
31+ </item>
32+</rdf:RDF>`
33+
34+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
35+ assert.NilError(t, err)
36+
37+ assert.Equal(t, result.Feed.URL, "https://example.com/feed.rdf")
38+ assert.Equal(t, result.Feed.Title, "Test Feed")
39+ assert.Equal(t, result.Feed.SiteURL, "https://example.com")
40+ assert.Equal(t, result.Feed.Description, "A test feed")
41+ assert.Equal(t, result.Feed.Type, "rdf")
42+ assert.Equal(t, len(result.Articles), 2)
43+
44+ assert.Equal(t, result.Articles[0].Title, "First Item")
45+ assert.Equal(t, result.Articles[0].URL, "https://example.com/1")
46+ assert.Equal(t, result.Articles[0].GUID, "https://example.com/1")
47+ assert.Equal(t, result.Articles[0].Author, "Alice")
48+ assert.Equal(t, result.Articles[0].Summary, "Description of first item")
49+ assert.Assert(t, !result.Articles[0].Published.IsZero())
50+
51+ assert.Equal(t, result.Articles[1].Title, "Second Item")
52+ assert.Equal(t, result.Articles[1].GUID, "https://example.com/2")
53+ assert.Equal(t, result.Articles[1].Author, "")
54+}
55+
56+func TestParse_RSS1_FallsBackToLinkForGUID(t *testing.T) {
57+ rdf := `<?xml version="1.0"?>
58+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
59+ <channel>
60+ <title>T</title>
61+ <link>https://example.com</link>
62+ </channel>
63+ <item>
64+ <title>No about attr</title>
65+ <link>https://example.com/noabout</link>
66+ </item>
67+</rdf:RDF>`
68+
69+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
70+ assert.NilError(t, err)
71+ assert.Equal(t, len(result.Articles), 1)
72+ assert.Equal(t, result.Articles[0].GUID, "https://example.com/noabout")
73+}
74+
75+func TestParse_RSS1_EmptyFeed(t *testing.T) {
76+ rdf := `<?xml version="1.0"?>
77+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
78+ <channel>
79+ <title>Empty</title>
80+ <link>https://example.com</link>
81+ </channel>
82+</rdf:RDF>`
83+
84+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
85+ assert.NilError(t, err)
86+ assert.Equal(t, result.Feed.Title, "Empty")
87+ assert.Equal(t, len(result.Articles), 0)
88+}
89+
90+func TestParse_RSS1_DateParsing(t *testing.T) {
91+ rdf := `<?xml version="1.0"?>
92+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:dc="http://purl.org/dc/elements/1.1/">
93+ <channel>
94+ <title>T</title>
95+ <link>https://example.com</link>
96+ </channel>
97+ <item rdf:about="https://example.com/1">
98+ <title>T</title>
99+ <link>https://example.com/1</link>
100+ <dc:date>2026-04-21T10:00:00+09:00</dc:date>
101+ </item>
102+</rdf:RDF>`
103+
104+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
105+ assert.NilError(t, err)
106+
107+ expected, _ := time.Parse(time.RFC3339, "2026-04-21T01:00:00Z")
108+ assert.Equal(t, result.Articles[0].Published.UTC(), expected.UTC())
109+}
110+
111+func TestParse_RSS1_WithContentEncoded(t *testing.T) {
112+ rdf := `<?xml version="1.0"?>
113+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/">
114+ <channel>
115+ <title>T</title>
116+ <link>https://example.com</link>
117+ </channel>
118+ <item rdf:about="https://example.com/1">
119+ <title>T</title>
120+ <link>https://example.com/1</link>
121+ <description>Summary text</description>
122+ <content:encoded><![CDATA[<p>Full content here</p>]]></content:encoded>
123+ <dc:date>2026-04-21T10:00:00Z</dc:date>
124+ </item>
125+</rdf:RDF>`
126+
127+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
128+ assert.NilError(t, err)
129+ assert.Equal(t, result.Articles[0].Content, "<p>Full content here</p>")
130+ assert.Equal(t, result.Articles[0].Summary, "Summary text")
131+}
132+
133+func TestParse_RSS2StillWorks(t *testing.T) {
134+ rss := `<?xml version="1.0"?>
135+<rss version="2.0">
136+ <channel>
137+ <title>RSS2 Feed</title>
138+ <link>https://example.com</link>
139+ <description>A test</description>
140+ <item>
141+ <title>Item</title>
142+ <link>https://example.com/1</link>
143+ <guid>https://example.com/1</guid>
144+ <pubDate>Mon, 21 Apr 2026 10:00:00 UTC</pubDate>
145+ </item>
146+ </channel>
147+</rss>`
148+
149+ result, err := Parse(strings.NewReader(rss), "https://example.com/feed.xml")
150+ assert.NilError(t, err)
151+ assert.Equal(t, result.Feed.Type, "rss")
152+ assert.Equal(t, len(result.Articles), 1)
153+}
154+
155+func TestParse_AtomStillWorks(t *testing.T) {
156+ atom := `<?xml version="1.0"?>
157+<feed xmlns="http://www.w3.org/2005/Atom">
158+ <title>Atom Feed</title>
159+ <link href="https://example.com" rel="alternate"/>
160+ <entry>
161+ <title>Entry</title>
162+ <link href="https://example.com/1" rel="alternate"/>
163+ <id>https://example.com/1</id>
164+ <updated>2026-04-21T10:00:00Z</updated>
165+ </entry>
166+</feed>`
167+
168+ result, err := Parse(strings.NewReader(atom), "https://example.com/feed.atom")
169+ assert.NilError(t, err)
170+ assert.Equal(t, result.Feed.Type, "atom")
171+ assert.Equal(t, len(result.Articles), 1)
172+}
new file mode 100644
@@ -0,0 +1,172 @@
1+package feed
2+
3+import (
4+ "strings"
5+ "testing"
6+ "time"
7+
8+ "gotest.tools/v3/assert"
9+)
10+
11+func TestParse_RSS1(t *testing.T) {
12+ rdf := `<?xml version="1.0" encoding="UTF-8"?>
13+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:dc="http://purl.org/dc/elements/1.1/">
14+ <channel>
15+ <title>Test Feed</title>
16+ <link>https://example.com</link>
17+ <description>A test feed</description>
18+ </channel>
19+ <item rdf:about="https://example.com/1">
20+ <title>First Item</title>
21+ <link>https://example.com/1</link>
22+ <dc:date>2026-04-21T10:00:00+09:00</dc:date>
23+ <dc:creator>Alice</dc:creator>
24+ <description>Description of first item</description>
25+ </item>
26+ <item rdf:about="https://example.com/2">
27+ <title>Second Item</title>
28+ <link>https://example.com/2</link>
29+ <dc:date>2026-04-20T08:00:00Z</dc:date>
30+ <description>Description of second item</description>
31+ </item>
32+</rdf:RDF>`
33+
34+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
35+ assert.NilError(t, err)
36+
37+ assert.Equal(t, result.Feed.URL, "https://example.com/feed.rdf")
38+ assert.Equal(t, result.Feed.Title, "Test Feed")
39+ assert.Equal(t, result.Feed.SiteURL, "https://example.com")
40+ assert.Equal(t, result.Feed.Description, "A test feed")
41+ assert.Equal(t, result.Feed.Type, "rdf")
42+ assert.Equal(t, len(result.Articles), 2)
43+
44+ assert.Equal(t, result.Articles[0].Title, "First Item")
45+ assert.Equal(t, result.Articles[0].URL, "https://example.com/1")
46+ assert.Equal(t, result.Articles[0].GUID, "https://example.com/1")
47+ assert.Equal(t, result.Articles[0].Author, "Alice")
48+ assert.Equal(t, result.Articles[0].Summary, "Description of first item")
49+ assert.Assert(t, !result.Articles[0].Published.IsZero())
50+
51+ assert.Equal(t, result.Articles[1].Title, "Second Item")
52+ assert.Equal(t, result.Articles[1].GUID, "https://example.com/2")
53+ assert.Equal(t, result.Articles[1].Author, "")
54+}
55+
56+func TestParse_RSS1_FallsBackToLinkForGUID(t *testing.T) {
57+ rdf := `<?xml version="1.0"?>
58+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
59+ <channel>
60+ <title>T</title>
61+ <link>https://example.com</link>
62+ </channel>
63+ <item>
64+ <title>No about attr</title>
65+ <link>https://example.com/noabout</link>
66+ </item>
67+</rdf:RDF>`
68+
69+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
70+ assert.NilError(t, err)
71+ assert.Equal(t, len(result.Articles), 1)
72+ assert.Equal(t, result.Articles[0].GUID, "https://example.com/noabout")
73+}
74+
75+func TestParse_RSS1_EmptyFeed(t *testing.T) {
76+ rdf := `<?xml version="1.0"?>
77+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
78+ <channel>
79+ <title>Empty</title>
80+ <link>https://example.com</link>
81+ </channel>
82+</rdf:RDF>`
83+
84+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
85+ assert.NilError(t, err)
86+ assert.Equal(t, result.Feed.Title, "Empty")
87+ assert.Equal(t, len(result.Articles), 0)
88+}
89+
90+func TestParse_RSS1_DateParsing(t *testing.T) {
91+ rdf := `<?xml version="1.0"?>
92+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:dc="http://purl.org/dc/elements/1.1/">
93+ <channel>
94+ <title>T</title>
95+ <link>https://example.com</link>
96+ </channel>
97+ <item rdf:about="https://example.com/1">
98+ <title>T</title>
99+ <link>https://example.com/1</link>
100+ <dc:date>2026-04-21T10:00:00+09:00</dc:date>
101+ </item>
102+</rdf:RDF>`
103+
104+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
105+ assert.NilError(t, err)
106+
107+ expected, _ := time.Parse(time.RFC3339, "2026-04-21T01:00:00Z")
108+ assert.Equal(t, result.Articles[0].Published.UTC(), expected.UTC())
109+}
110+
111+func TestParse_RSS1_WithContentEncoded(t *testing.T) {
112+ rdf := `<?xml version="1.0"?>
113+<rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/">
114+ <channel>
115+ <title>T</title>
116+ <link>https://example.com</link>
117+ </channel>
118+ <item rdf:about="https://example.com/1">
119+ <title>T</title>
120+ <link>https://example.com/1</link>
121+ <description>Summary text</description>
122+ <content:encoded><![CDATA[<p>Full content here</p>]]></content:encoded>
123+ <dc:date>2026-04-21T10:00:00Z</dc:date>
124+ </item>
125+</rdf:RDF>`
126+
127+ result, err := Parse(strings.NewReader(rdf), "https://example.com/feed.rdf")
128+ assert.NilError(t, err)
129+ assert.Equal(t, result.Articles[0].Content, "<p>Full content here</p>")
130+ assert.Equal(t, result.Articles[0].Summary, "Summary text")
131+}
132+
133+func TestParse_RSS2StillWorks(t *testing.T) {
134+ rss := `<?xml version="1.0"?>
135+<rss version="2.0">
136+ <channel>
137+ <title>RSS2 Feed</title>
138+ <link>https://example.com</link>
139+ <description>A test</description>
140+ <item>
141+ <title>Item</title>
142+ <link>https://example.com/1</link>
143+ <guid>https://example.com/1</guid>
144+ <pubDate>Mon, 21 Apr 2026 10:00:00 UTC</pubDate>
145+ </item>
146+ </channel>
147+</rss>`
148+
149+ result, err := Parse(strings.NewReader(rss), "https://example.com/feed.xml")
150+ assert.NilError(t, err)
151+ assert.Equal(t, result.Feed.Type, "rss")
152+ assert.Equal(t, len(result.Articles), 1)
153+}
154+
155+func TestParse_AtomStillWorks(t *testing.T) {
156+ atom := `<?xml version="1.0"?>
157+<feed xmlns="http://www.w3.org/2005/Atom">
158+ <title>Atom Feed</title>
159+ <link href="https://example.com" rel="alternate"/>
160+ <entry>
161+ <title>Entry</title>
162+ <link href="https://example.com/1" rel="alternate"/>
163+ <id>https://example.com/1</id>
164+ <updated>2026-04-21T10:00:00Z</updated>
165+ </entry>
166+</feed>`
167+
168+ result, err := Parse(strings.NewReader(atom), "https://example.com/feed.atom")
169+ assert.NilError(t, err)
170+ assert.Equal(t, result.Feed.Type, "atom")
171+ assert.Equal(t, len(result.Articles), 1)
172+}