diff --git a/downloader.go b/downloader.go index 503fa71..90a5a1c 100644 --- a/downloader.go +++ b/downloader.go @@ -19,24 +19,14 @@ import ( // Downloader is an asynchronous downloader type Downloader struct { URL string - Done chan struct{} Resp *http.Response out *os.File completed int64 completedLock sync.Mutex size int64 err error -} - -// Close the download -func (d *Downloader) close() { - if d.out != nil { - d.out.Close() - } - if d.Resp != nil { - d.Resp.Body.Close() - } - close(d.Done) + timeout time.Duration + wdog watchdog } // Size return the size of the download (or -1 if the server doesn't provide it) @@ -50,14 +40,17 @@ func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration t := time.NewTicker(interval) defer t.Stop() - go d.Run() + res := make(chan error, 1) + go func() { + res <- d.Run() + }() for { select { case <-t.C: poll(d.Completed()) - case <-d.Done: + case err := <-res: poll(d.Completed()) - return d.Error() + return err } } } @@ -66,8 +59,6 @@ func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration // This method can be run in a goroutine to perform an asynchronous download; // it will close the Done channel when the download is completed or an error occurs. func (d *Downloader) Run() error { - defer d.close() - d.completedLock.Lock() skip := (d.completed == d.size) d.completedLock.Unlock() @@ -75,6 +66,12 @@ func (d *Downloader) Run() error { return d.Error() } + defer func() { + d.out.Close() + d.Resp.Body.Close() + d.wdog.Cancel() + }() + in := d.Resp.Body buff := [4096]byte{} for { @@ -84,6 +81,8 @@ func (d *Downloader) Run() error { d.completedLock.Lock() d.completed += int64(n) d.completedLock.Unlock() + // Extend inactivity timeout deadline + d.wdog.Kick() } if err == io.EOF { break @@ -124,6 +123,31 @@ func DownloadWithConfig(file string, reqURL string, config Config) (*Downloader, return DownloadWithConfigAndContext(context.Background(), file, reqURL, config) } +func doHeadRequest(ctx context.Context, reqURL string, config Config) (*http.Response, error) { + headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("setting up HEAD request: %s", err) + } + if config.ExtraHeaders != nil { + for k, v := range config.ExtraHeaders { + headReq.Header.Set(k, v) + } + } + // Should complete the HEAD call within the inactivity timeout + config.HttpClient.Timeout = config.InactivityTimeout + headResp, err := config.HttpClient.Do(headReq) + if err != nil { + return nil, fmt.Errorf("performing HEAD request: %s", err) + } + if _, err := io.Copy(io.Discard, headResp.Body); err != nil { + return nil, err + } + if err := headResp.Body.Close(); err != nil { + return nil, err + } + return headResp, nil +} + // DownloadWithConfigAndContext applies an additional configuration to the http client and // returns an asynchronous downloader that will download the specified url // in the specified file. @@ -140,27 +164,18 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin } // Perform a HEAD call to gather information about the server capabilities and remote file - headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil) + headResp, err := doHeadRequest(ctx, reqURL, config) if err != nil { - return nil, fmt.Errorf("setting up HEAD request: %s", err) - } - if config.ExtraHeaders != nil { - for k, v := range config.ExtraHeaders { - headReq.Header.Set(k, v) - } - } - headResp, err := config.HttpClient.Do(headReq) - if err != nil { - return nil, fmt.Errorf("performing HEAD request: %s", err) + return nil, err } remoteSize := headResp.ContentLength // -1 if server doesn't send Content-Length serverCanResume := (headResp.Header.Get("Accept-Ranges") == "bytes") && (remoteSize != -1) + + // Perform acceptance checks var acceptError error if config.AcceptFunc != nil { acceptError = config.AcceptFunc(headResp) } - _, _ = io.Copy(io.Discard, headResp.Body) - _ = headResp.Body.Close() if acceptError != nil { return nil, acceptError } @@ -172,7 +187,6 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin // Size matches: assume the file is already downloaded return &Downloader{ URL: reqURL, - Done: make(chan struct{}), Resp: headResp, completed: remoteSize, size: remoteSize, @@ -186,8 +200,11 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin } // Perform the actual GET request + // Setup inactivity timeout for the GET request + ctx, wdog := newWatchdog(ctx, config.InactivityTimeout) req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { + wdog.Cancel() return nil, fmt.Errorf("setting up HTTP request: %s", err) } if config.ExtraHeaders != nil { @@ -201,6 +218,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin } resp, err := config.HttpClient.Do(req) if err != nil { + wdog.Cancel() return nil, err } @@ -208,6 +226,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin if !config.DoNotErrorOnNon2xxStatusCode { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { _ = resp.Body.Close() + wdog.Cancel() return &Downloader{ URL: reqURL, Resp: resp, // Return response for further inspection @@ -225,15 +244,16 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin f, err := os.OpenFile(file, flags, 0644) if err != nil { _ = resp.Body.Close() + wdog.Cancel() return nil, fmt.Errorf("opening %s for writing: %s", file, err) } return &Downloader{ URL: reqURL, - Done: make(chan struct{}), Resp: resp, out: f, completed: completed, size: remoteSize, + wdog: wdog, }, nil } diff --git a/downloader_config.go b/downloader_config.go index f954af7..b41c396 100644 --- a/downloader_config.go +++ b/downloader_config.go @@ -9,6 +9,7 @@ package downloader import ( "net/http" "sync" + "time" ) // Config contains the configuration for the downloader @@ -26,6 +27,9 @@ type Config struct { // DoNotErrorOnNon2xxStatusCode set to true to not return an error // if the server returns a non-2xx status code. DoNotErrorOnNon2xxStatusCode bool + // InactivityTimeout is the duration after which, if no data is received, + // the download is aborted. If set to 0, no timeout is applied. + InactivityTimeout time.Duration } var defaultConfig Config = Config{} diff --git a/downloader_test.go b/downloader_test.go index dc690b6..4015215 100644 --- a/downloader_test.go +++ b/downloader_test.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "net/http" + "net/http/httptest" "os" "path/filepath" "testing" @@ -145,9 +146,9 @@ func TestRunAndPool(t *testing.T) { prevCurr = curr callCount++ } - _ = d.RunAndPoll(callback, time.Millisecond) + require.NoError(t, d.RunAndPoll(callback, time.Millisecond)) fmt.Printf("callback called %d times\n", callCount) - require.True(t, callCount > 10) + require.Greater(t, callCount, 10) require.Equal(t, int64(4897949), d.Completed()) } @@ -197,8 +198,12 @@ func TestApplyUserAgentHeaderUsingConfig(t *testing.T) { func TestContextCancelation(t *testing.T) { slowHandler := func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("Content-Length", "500") + return + } for i := range 50 { - fmt.Fprintf(w, "Hello %d\n", i) + fmt.Fprintf(w, "Hello %02d.\n", i) w.(http.Flusher).Flush() time.Sleep(100 * time.Millisecond) } @@ -206,18 +211,13 @@ func TestContextCancelation(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/slow", slowHandler) - server := &http.Server{Addr: ":8080", Handler: mux} - go func() { - err := server.ListenAndServe() - fmt.Println("Server stopped with err:", err) - }() - // Wait for server start - time.Sleep(time.Second) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) tmpFile := makeTmpFile(t) ctx, cancel := context.WithCancel(context.Background()) - d, err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, "http://127.0.0.1:8080/slow", downloader.Config{}) + d, err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, server.URL+"/slow", downloader.Config{}) require.NoError(t, err) // Cancel in two seconds @@ -234,7 +234,117 @@ func TestContextCancelation(t *testing.T) { max = curr }, 100*time.Millisecond) require.EqualError(t, err, "context canceled") - require.True(t, max < 400) + require.True(t, max < 210) + fmt.Println("Context canceled successfully") +} + +func TestTimeoutOnHEADCall(t *testing.T) { + timeoutHandler := func(w http.ResponseWriter, r *http.Request) { + time.Sleep(3 * time.Second) + fmt.Fprintln(w, "This is a slow response") + } + mux := http.NewServeMux() + mux.HandleFunc("/timeout", timeoutHandler) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + + tmpFile := makeTmpFile(t) + + config := downloader.Config{ + InactivityTimeout: 1 * time.Second, + } + + startTime := time.Now() + d, err := downloader.DownloadWithConfig(tmpFile, server.URL+"/timeout", config) + require.Error(t, err) + require.Nil(t, d) + require.Contains(t, err.Error(), "context deadline exceeded") + elapsed := time.Since(startTime) + require.True(t, elapsed < 5*time.Second) + fmt.Println("Download aborted due to timeout as expected after", elapsed) +} + +func TestTimeoutOnGETCall(t *testing.T) { + slowHandler := func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("Content-Length", "5000") + return + } + time.Sleep(3 * time.Second) // Delay response sending any data + } + mux := http.NewServeMux() + mux.HandleFunc("/slow", slowHandler) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + + tmpFile := makeTmpFile(t) + + config := downloader.Config{ + InactivityTimeout: 1 * time.Second, + } + + startTime := time.Now() + d, err := downloader.DownloadWithConfig(tmpFile, server.URL+"/slow", config) + require.Error(t, err) + require.Nil(t, d) + require.Contains(t, err.Error(), "i/o timeout") + elapsed := time.Since(startTime) + require.True(t, elapsed < 5*time.Second) + fmt.Println("Download aborted due to timeout as expected after", elapsed) +} + +func TestInactivityTimeout(t *testing.T) { + // Create a handler that sends some data, then becomes inactive + inactiveHandler := func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + // For HEAD requests, just return Content-Length + w.Header().Set("Content-Length", "50") + return + } + w.(http.Flusher).Flush() // Ensure headers are sent immediately + + // Send initial burst of data (500 ms) + for range 5 { + time.Sleep(100 * time.Millisecond) + fmt.Fprintf(w, "AAAAA") + w.(http.Flusher).Flush() + } + // Rest inactive for 1000 ms + time.Sleep(1000 * time.Millisecond) + // Send more data (500 ms) + for range 5 { + time.Sleep(100 * time.Millisecond) + fmt.Fprintf(w, "AAAAA") + w.(http.Flusher).Flush() + } + // 2000 ms total time + } + + // Start test server + mux := http.NewServeMux() + mux.HandleFunc("/inactive", inactiveHandler) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + + // Request download with inactivity timeout of 500ms + tmpFile := makeTmpFile(t) + d, err := downloader.DownloadWithConfigAndContext( + t.Context(), tmpFile, server.URL+"/inactive", + downloader.Config{ + InactivityTimeout: 500 * time.Millisecond, + }, + ) + require.NoError(t, err) + + // Run the download, should timeout due to inactivity + startTime := time.Now() + err = d.Run() + elapsed := time.Since(startTime) + + // Check that we got a timeout error + require.Error(t, err) + require.Contains(t, err.Error(), "i/o timeout") - require.NoError(t, server.Shutdown(context.Background())) + // Check that it took around 1 second (initial burst of data 500ms + 500ms second timeout) + require.InEpsilon(t, 1000*time.Millisecond, elapsed, 0.05, "Elapsed time should be around 1 second, but is %v", elapsed) } diff --git a/watchdog.go b/watchdog.go new file mode 100644 index 0000000..3d166f3 --- /dev/null +++ b/watchdog.go @@ -0,0 +1,50 @@ +// +// Copyright 2025 Cristian Maglie. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// + +package downloader + +import ( + "context" + "os" + "time" +) + +type watchdog struct { + ctx context.Context + cancel context.CancelCauseFunc + timer *time.Timer + timeout time.Duration +} + +func newWatchdog(parent context.Context, timeout time.Duration) (context.Context, watchdog) { + ctx, cancel := context.WithCancelCause(parent) + var timer *time.Timer + if timeout > 0 { + timer = time.AfterFunc(timeout, func() { + // Cancel the context with a clear, standard error. + cancel(os.ErrDeadlineExceeded) + }) + } + return ctx, watchdog{ + ctx: ctx, + cancel: cancel, + timer: timer, + timeout: timeout, + } +} + +func (wd *watchdog) Kick() { + if wd.timeout > 0 { + wd.timer.Reset(wd.timeout) + } +} + +func (wd *watchdog) Cancel() { + if wd.timeout > 0 { + wd.timer.Stop() + } + wd.cancel(nil) +}