rickub
nandi/gleanpublic⑂ Fork 0
⑂ 83e5d54
Branches
claude/modal-container-min-76527a main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/glean.git
git clone ssh://git@rickub.com/nandi/glean.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Fix XML feed parsing with proper charset handlingUnverified

Julien Robert committed 2026-04-21T14:08:10+02:00 Browse files
83e5d54 parent: 23e1fce
modified go.mod +1 -0
@@ -43,6 +43,7 @@ require (
4343 gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
4444 golang.org/x/crypto v0.50.0 // indirect
4545 golang.org/x/sys v0.43.0 // indirect
46+ golang.org/x/text v0.36.0 // indirect
4647 golang.org/x/time v0.5.0 // indirect
4748 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
4849 google.golang.org/protobuf v1.34.2 // indirect
@@ -43,6 +43,7 @@ require (
43 gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect43 gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
44 golang.org/x/crypto v0.50.0 // indirect44 golang.org/x/crypto v0.50.0 // indirect
45 golang.org/x/sys v0.43.0 // indirect45 golang.org/x/sys v0.43.0 // indirect
46+ golang.org/x/text v0.36.0 // indirect
46 golang.org/x/time v0.5.0 // indirect47 golang.org/x/time v0.5.0 // indirect
47 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect48 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
48 google.golang.org/protobuf v1.34.2 // indirect49 google.golang.org/protobuf v1.34.2 // indirect
modified go.sum +2 -0
@@ -78,6 +78,8 @@ golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
7878 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
7979 golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
8080 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
81+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
82+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
8183 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
8284 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
8385 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -78,6 +78,8 @@ golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
78 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=78 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
79 golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=79 golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
80 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=80 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
81+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
82+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
81 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=83 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
82 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=84 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
83 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=85 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
modified internal/feed/parser.go +13 -2
@@ -7,6 +7,8 @@ import (
77 "io"
88 "strings"
99 "time"
10+
11+ htmlcharset "golang.org/x/net/html/charset"
1012 )
1113
1214 type Feed struct {
@@ -153,16 +155,25 @@ func parseJSONFeed(data []byte, feedURL string) (*ParseResult, error) {
153155 return result, nil
154156 }
155157
158+func makeXMLDecoder(data []byte) *xml.Decoder {
159+ decoder := xml.NewDecoder(strings.NewReader(string(data)))
160+ decoder.Strict = false
161+ decoder.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) {
162+ return htmlcharset.NewReader(input, "text/xml; charset="+charset)
163+ }
164+ return decoder
165+}
166+
156167 func parseXMLFeed(data []byte, feedURL string) (*ParseResult, error) {
157168 var rss rssFeed
158- if err := xml.Unmarshal(data, &rss); err == nil {
169+ if err := makeXMLDecoder(data).Decode(&rss); err == nil {
159170 if rss.XMLName.Local == "rss" {
160171 return convertRSS(&rss, feedURL), nil
161172 }
162173 }
163174
164175 var atom atomFeed
165- if err := xml.Unmarshal(data, &atom); err == nil {
176+ if err := makeXMLDecoder(data).Decode(&atom); err == nil {
166177 if atom.XMLName.Local == "feed" {
167178 return convertAtom(&atom, feedURL), nil
168179 }
@@ -7,6 +7,8 @@ import (
7 "io"7 "io"
8 "strings"8 "strings"
9 "time"9 "time"
10+
11+ htmlcharset "golang.org/x/net/html/charset"
10 )12 )
11 13
12 type Feed struct {14 type Feed struct {
@@ -153,16 +155,25 @@ func parseJSONFeed(data []byte, feedURL string) (*ParseResult, error) {
153 return result, nil155 return result, nil
154 }156 }
155 157
158+func makeXMLDecoder(data []byte) *xml.Decoder {
159+ decoder := xml.NewDecoder(strings.NewReader(string(data)))
160+ decoder.Strict = false
161+ decoder.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) {
162+ return htmlcharset.NewReader(input, "text/xml; charset="+charset)
163+ }
164+ return decoder
165+}
166+
156 func parseXMLFeed(data []byte, feedURL string) (*ParseResult, error) {167 func parseXMLFeed(data []byte, feedURL string) (*ParseResult, error) {
157 var rss rssFeed168 var rss rssFeed
158- if err := xml.Unmarshal(data, &rss); err == nil {169+ if err := makeXMLDecoder(data).Decode(&rss); err == nil {
159 if rss.XMLName.Local == "rss" {170 if rss.XMLName.Local == "rss" {
160 return convertRSS(&rss, feedURL), nil171 return convertRSS(&rss, feedURL), nil
161 }172 }
162 }173 }
163 174
164 var atom atomFeed175 var atom atomFeed
165- if err := xml.Unmarshal(data, &atom); err == nil {176+ if err := makeXMLDecoder(data).Decode(&atom); err == nil {
166 if atom.XMLName.Local == "feed" {177 if atom.XMLName.Local == "feed" {
167 return convertAtom(&atom, feedURL), nil178 return convertAtom(&atom, feedURL), nil
168 }179 }