Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 66 additions & 20 deletions downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@ import (
// Downloader is an asynchronous downloader
type Downloader struct {
URL string
Done chan struct{}
Resp *http.Response
out *os.File
completed int64
completedLock sync.Mutex
size int64
err error
timeout time.Duration
cancel context.CancelCauseFunc
}

// Close the download
Expand All @@ -36,7 +37,9 @@ func (d *Downloader) close() {
if d.Resp != nil {
d.Resp.Body.Close()
}
close(d.Done)
if d.cancel != nil {
d.cancel(nil)
}
}

// Size return the size of the download (or -1 if the server doesn't provide it)
Expand All @@ -50,14 +53,17 @@ func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration
t := time.NewTicker(interval)
defer t.Stop()

go d.Run()
res := make(chan error, 1)
go func() {
res <- d.Run()
}()
for {
select {
case <-t.C:
poll(d.Completed())
case <-d.Done:
case err := <-res:
poll(d.Completed())
return d.Error()
return err
}
}
}
Expand All @@ -76,6 +82,14 @@ func (d *Downloader) Run() error {
}

in := d.Resp.Body
var timeoutTimer *time.Timer
if d.timeout > 0 {
timeoutTimer = time.AfterFunc(d.timeout, func() {
d.cancel(os.ErrDeadlineExceeded)
})
defer timeoutTimer.Stop()
}

buff := [4096]byte{}
for {
n, err := in.Read(buff[:])
Expand All @@ -84,6 +98,10 @@ func (d *Downloader) Run() error {
d.completedLock.Lock()
d.completed += int64(n)
d.completedLock.Unlock()
if d.timeout > 0 {
// Extend inactivity timeout deadline
timeoutTimer.Reset(d.timeout)
}
}
if err == io.EOF {
break
Expand Down Expand Up @@ -124,6 +142,31 @@ func DownloadWithConfig(file string, reqURL string, config Config) (*Downloader,
return DownloadWithConfigAndContext(context.Background(), file, reqURL, config)
}

func doHeadRequest(ctx context.Context, reqURL string, config Config) (*http.Response, error) {
headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil)
if err != nil {
return nil, fmt.Errorf("setting up HEAD request: %s", err)
}
if config.ExtraHeaders != nil {
for k, v := range config.ExtraHeaders {
headReq.Header.Set(k, v)
}
}
// Should complete the HEAD call within the inactivity timeout
config.HttpClient.Timeout = config.InactivityTimeout
headResp, err := config.HttpClient.Do(headReq)
if err != nil {
return nil, fmt.Errorf("performing HEAD request: %s", err)
}
if _, err := io.Copy(io.Discard, headResp.Body); err != nil {
return nil, err
}
if err := headResp.Body.Close(); err != nil {
return nil, err
}
return headResp, nil
}

// DownloadWithConfigAndContext applies an additional configuration to the http client and
// returns an asynchronous downloader that will download the specified url
// in the specified file.
Expand All @@ -140,27 +183,18 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
}

// Perform a HEAD call to gather information about the server capabilities and remote file
headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil)
if err != nil {
return nil, fmt.Errorf("setting up HEAD request: %s", err)
}
if config.ExtraHeaders != nil {
for k, v := range config.ExtraHeaders {
headReq.Header.Set(k, v)
}
}
headResp, err := config.HttpClient.Do(headReq)
headResp, err := doHeadRequest(ctx, reqURL, config)
if err != nil {
return nil, fmt.Errorf("performing HEAD request: %s", err)
return nil, err
}
remoteSize := headResp.ContentLength // -1 if server doesn't send Content-Length
serverCanResume := (headResp.Header.Get("Accept-Ranges") == "bytes") && (remoteSize != -1)

// Perform acceptance checks
var acceptError error
if config.AcceptFunc != nil {
acceptError = config.AcceptFunc(headResp)
}
_, _ = io.Copy(io.Discard, headResp.Body)
_ = headResp.Body.Close()
if acceptError != nil {
return nil, acceptError
}
Expand All @@ -172,7 +206,6 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
// Size matches: assume the file is already downloaded
return &Downloader{
URL: reqURL,
Done: make(chan struct{}),
Resp: headResp,
completed: remoteSize,
size: remoteSize,
Expand All @@ -186,8 +219,17 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
}

// Perform the actual GET request
ctx, cancel := context.WithCancelCause(ctx)
// Setup inactivity timeout for the GET request
if config.InactivityTimeout > 0 {
timer := time.AfterFunc(config.InactivityTimeout, func() {
cancel(os.ErrDeadlineExceeded)
})
defer timer.Stop()
Comment thread
cmaglie marked this conversation as resolved.
Resolved
}
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
if err != nil {
cancel(nil)
return nil, fmt.Errorf("setting up HTTP request: %s", err)
}
if config.ExtraHeaders != nil {
Expand All @@ -201,13 +243,15 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
}
resp, err := config.HttpClient.Do(req)
if err != nil {
cancel(nil)
return nil, err
}

// Check server response
if !config.DoNotErrorOnNon2xxStatusCode {
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
_ = resp.Body.Close()
cancel(nil)
return &Downloader{
URL: reqURL,
Resp: resp, // Return response for further inspection
Expand All @@ -225,15 +269,17 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
f, err := os.OpenFile(file, flags, 0644)
if err != nil {
_ = resp.Body.Close()
cancel(nil)
return nil, fmt.Errorf("opening %s for writing: %s", file, err)
}

return &Downloader{
URL: reqURL,
Done: make(chan struct{}),
Resp: resp,
out: f,
completed: completed,
size: remoteSize,
cancel: cancel,
timeout: config.InactivityTimeout,
}, nil
}
4 changes: 4 additions & 0 deletions downloader_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ package downloader
import (
"net/http"
"sync"
"time"
)

// Config contains the configuration for the downloader
Expand All @@ -26,6 +27,9 @@ type Config struct {
// DoNotErrorOnNon2xxStatusCode set to true to not return an error
// if the server returns a non-2xx status code.
DoNotErrorOnNon2xxStatusCode bool
// InactivityTimeout is the duration after which, if no data is received,
// the download is aborted. If set to 0, no timeout is applied.
InactivityTimeout time.Duration
}

var defaultConfig Config = Config{}
Expand Down
136 changes: 123 additions & 13 deletions downloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
Expand Down Expand Up @@ -145,9 +146,9 @@ func TestRunAndPool(t *testing.T) {
prevCurr = curr
callCount++
}
_ = d.RunAndPoll(callback, time.Millisecond)
require.NoError(t, d.RunAndPoll(callback, time.Millisecond))
fmt.Printf("callback called %d times\n", callCount)
require.True(t, callCount > 10)
require.Greater(t, callCount, 10)
require.Equal(t, int64(4897949), d.Completed())
}

Expand Down Expand Up @@ -197,27 +198,26 @@ func TestApplyUserAgentHeaderUsingConfig(t *testing.T) {

func TestContextCancelation(t *testing.T) {
slowHandler := func(w http.ResponseWriter, r *http.Request) {
if r.Method == "HEAD" {
w.Header().Set("Content-Length", "500")
return
}
for i := range 50 {
fmt.Fprintf(w, "Hello %d\n", i)
fmt.Fprintf(w, "Hello %02d.\n", i)
w.(http.Flusher).Flush()
time.Sleep(100 * time.Millisecond)
}
}

mux := http.NewServeMux()
mux.HandleFunc("/slow", slowHandler)
server := &http.Server{Addr: ":8080", Handler: mux}
go func() {
err := server.ListenAndServe()
fmt.Println("Server stopped with err:", err)
}()
// Wait for server start
time.Sleep(time.Second)
server := httptest.NewServer(mux)
t.Cleanup(server.Close)

tmpFile := makeTmpFile(t)

ctx, cancel := context.WithCancel(context.Background())
d, err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, "http://127.0.0.1:8080/slow", downloader.Config{})
d, err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, server.URL+"/slow", downloader.Config{})
require.NoError(t, err)

// Cancel in two seconds
Expand All @@ -234,7 +234,117 @@ func TestContextCancelation(t *testing.T) {
max = curr
}, 100*time.Millisecond)
require.EqualError(t, err, "context canceled")
require.True(t, max < 400)
require.True(t, max < 210)
fmt.Println("Context canceled successfully")
}

func TestTimeoutOnHEADCall(t *testing.T) {
timeoutHandler := func(w http.ResponseWriter, r *http.Request) {
time.Sleep(3 * time.Second)
fmt.Fprintln(w, "This is a slow response")
}
mux := http.NewServeMux()
mux.HandleFunc("/timeout", timeoutHandler)
server := httptest.NewServer(mux)
t.Cleanup(server.Close)

tmpFile := makeTmpFile(t)

config := downloader.Config{
InactivityTimeout: 1 * time.Second,
}

startTime := time.Now()
d, err := downloader.DownloadWithConfig(tmpFile, server.URL+"/timeout", config)
require.Error(t, err)
require.Nil(t, d)
require.Contains(t, err.Error(), "context deadline exceeded")
elapsed := time.Since(startTime)
require.True(t, elapsed < 5*time.Second)
fmt.Println("Download aborted due to timeout as expected after", elapsed)
}

func TestTimeoutOnGETCall(t *testing.T) {
slowHandler := func(w http.ResponseWriter, r *http.Request) {
if r.Method == "HEAD" {
w.Header().Set("Content-Length", "5000")
return
}
time.Sleep(3 * time.Second) // Delay response sending any data
}
mux := http.NewServeMux()
mux.HandleFunc("/slow", slowHandler)
server := httptest.NewServer(mux)
t.Cleanup(server.Close)

tmpFile := makeTmpFile(t)

config := downloader.Config{
InactivityTimeout: 1 * time.Second,
}

startTime := time.Now()
d, err := downloader.DownloadWithConfig(tmpFile, server.URL+"/slow", config)
require.Error(t, err)
require.Nil(t, d)
require.Contains(t, err.Error(), "i/o timeout")
elapsed := time.Since(startTime)
require.True(t, elapsed < 5*time.Second)
fmt.Println("Download aborted due to timeout as expected after", elapsed)
}

func TestInactivityTimeout(t *testing.T) {
// Create a handler that sends some data, then becomes inactive
inactiveHandler := func(w http.ResponseWriter, r *http.Request) {
if r.Method == "HEAD" {
// For HEAD requests, just return Content-Length
w.Header().Set("Content-Length", "50")
return
}
w.(http.Flusher).Flush() // Ensure headers are sent immediately

// Send initial burst of data (500 ms)
for range 5 {
time.Sleep(100 * time.Millisecond)
fmt.Fprintf(w, "AAAAA")
w.(http.Flusher).Flush()
}
// Rest inactive for 1000 ms
time.Sleep(1000 * time.Millisecond)
// Send more data (500 ms)
for range 5 {
time.Sleep(100 * time.Millisecond)
fmt.Fprintf(w, "AAAAA")
w.(http.Flusher).Flush()
}
// 2000 ms total time
}

// Start test server
mux := http.NewServeMux()
mux.HandleFunc("/inactive", inactiveHandler)
server := httptest.NewServer(mux)
t.Cleanup(server.Close)

// Request download with inactivity timeout of 500ms
tmpFile := makeTmpFile(t)
d, err := downloader.DownloadWithConfigAndContext(
t.Context(), tmpFile, server.URL+"/inactive",
downloader.Config{
InactivityTimeout: 500 * time.Millisecond,
},
)
require.NoError(t, err)

// Run the download, should timeout due to inactivity
startTime := time.Now()
err = d.Run()
elapsed := time.Since(startTime)

// Check that we got a timeout error
require.Error(t, err)
require.Contains(t, err.Error(), "i/o timeout")

require.NoError(t, server.Shutdown(context.Background()))
// Check that it took around 1 second (initial burst of data 500ms + 500ms second timeout)
require.InEpsilon(t, 1000*time.Millisecond, elapsed, 0.05, "Elapsed time should be around 1 second, but is %v", elapsed)
}