nandi/glean · 6be8ad9

nandi/gleanpublic⑂ Fork 0

⑂ 6be8ad9

Tune HTTP client timeouts and use Cloudflare DNSUnverified

Julien Robert committed 2026-04-23T22:01:54+02:00 Browse files

6be8ad9 parent: 12e3cec

modified internal/feed/fetcher.go +2 -2 Viewed

@@ -25,10 +25,10 @@ type Fetcher struct {
25	25	func NewFetcher() *Fetcher {
26	26	return &Fetcher{
27	27	httpClient: &http.Client{
28		- Timeout: 30 * time.Second,
	28	+ Timeout: 10 * time.Second,
29	29	Transport: httpclient.NewTransport(),
30	30	CheckRedirect: func(req http.Request, via []http.Request) error {
31		- if len(via) >= 10 {
	31	+ if len(via) >= 5 {
32	32	return fmt.Errorf("too many redirects")
33	33	}
34	34	return nil

  func NewFetcher() *Fetcher {
  	return &Fetcher{
  		httpClient: &http.Client{
--			Timeout:   30 * time.Second,
++			Timeout:   10 * time.Second,
  			Transport: httpclient.NewTransport(),
  			CheckRedirect: func(req *http.Request, via []*http.Request) error {
--				if len(via) >= 10 {
++				if len(via) >= 5 {
  					return fmt.Errorf("too many redirects")
  				}
  				return nil

modified internal/httpclient/httpclient.go +17 -6 Viewed

@@ -16,16 +16,27 @@ const (
16	16	AcceptHTML = "text/html,application/xhtml+xml;q=0.9"
17	17	)
18	18
	19	+var dnsResolver = &net.Resolver{
	20	+ PreferGo: true,
	21	+ Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
	22	+ d := net.Dialer{Timeout: 3 * time.Second}
	23	+ return d.DialContext(ctx, "udp", "1.1.1.1:53")
	24	+ },
	25	+}
	26	+
19	27	func NewTransport() *http.Transport {
	28	+ dialer := &net.Dialer{
	29	+ Timeout: 5 * time.Second,
	30	+ KeepAlive: 15 * time.Second,
	31	+ FallbackDelay: 300 * time.Millisecond,
	32	+ Resolver: dnsResolver,
	33	+ }
	34	+
20	35	return &http.Transport{
21		- DialContext: (&net.Dialer{
22		- Timeout: 10 * time.Second,
23		- KeepAlive: 15 * time.Second,
24		- FallbackDelay: 300 * time.Millisecond,
25		- }).DialContext,
	36	+ DialContext: dialer.DialContext,
26	37	MaxIdleConns: 50,
27	38	IdleConnTimeout: 10 * time.Second,
28		- TLSHandshakeTimeout: 10 * time.Second,
	39	+ TLSHandshakeTimeout: 5 * time.Second,
29	40	ForceAttemptHTTP2: true,
30	41	}
31	42	}

  	AcceptHTML = "text/html,application/xhtml+xml;q=0.9"
  )
++var dnsResolver = &net.Resolver{
++	PreferGo: true,
++	Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
++		d := net.Dialer{Timeout: 3 * time.Second}
++		return d.DialContext(ctx, "udp", "1.1.1.1:53")
++	},
++}
++
  func NewTransport() *http.Transport {
++	dialer := &net.Dialer{
++		Timeout:       5 * time.Second,
++		KeepAlive:     15 * time.Second,
++		FallbackDelay: 300 * time.Millisecond,
++		Resolver:      dnsResolver,
++	}
++
  	return &http.Transport{
--		DialContext: (&net.Dialer{
++		DialContext:         dialer.DialContext,
--			Timeout:         10 * time.Second,
--			KeepAlive:       15 * time.Second,
--			FallbackDelay:   300 * time.Millisecond,
--		}).DialContext,
  		MaxIdleConns:        50,
  		IdleConnTimeout:     10 * time.Second,
--		TLSHandshakeTimeout: 10 * time.Second,
++		TLSHandshakeTimeout: 5 * time.Second,
  		ForceAttemptHTTP2:   true,
  	}
  }