From fb23f0922f0a01bb35f12c36e36255176b3ef386 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 9 Jan 2026 20:06:14 +0100 Subject: [PATCH 1/7] Remove RunAndPoll and put callbacks in Configuration --- downloader.go | 39 +++++++++++++++++++--------------- downloader_config.go | 5 +++++ downloader_test.go | 31 +++++++++++++++++---------- examples/download_file/main.go | 12 ++++++----- 4 files changed, 54 insertions(+), 33 deletions(-) diff --git a/downloader.go b/downloader.go index bed25be..07c19f6 100644 --- a/downloader.go +++ b/downloader.go @@ -26,6 +26,8 @@ type Downloader struct { size int64 err error wdog watchdog + pollCB func(current, size int64) + pollInterval time.Duration } // Size return the size of the download (or -1 if the server doesn't provide it) @@ -33,31 +35,32 @@ func (d *Downloader) Size() int64 { return d.size } -// RunAndPoll starts the downloader copy-loop and calls the poll function every -// interval time to update progress. -func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration) error { - t := time.NewTicker(interval) +// Run starts the downloader and waits until it completes the download. +func (d *Downloader) Run() error { + if d.pollCB == nil { + // Run the non-interactive download + return d.run() + } + + t := time.NewTicker(d.pollInterval) defer t.Stop() res := make(chan error, 1) go func() { - res <- d.Run() + res <- d.run() }() for { select { case <-t.C: - poll(d.Completed()) + d.pollCB(d.Completed(), d.size) case err := <-res: - poll(d.Completed()) + d.pollCB(d.Completed(), d.size) return err } } } -// Run starts the downloader and waits until it completes the download. -// This method can be run in a goroutine to perform an asynchronous download; -// it will close the Done channel when the download is completed or an error occurs. -func (d *Downloader) Run() error { +func (d *Downloader) run() error { d.completedLock.Lock() skip := (d.completed == d.size) d.completedLock.Unlock() @@ -257,11 +260,13 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin } return &Downloader{ - URL: reqURL, - Resp: resp, - out: f, - completed: completed, - size: remoteSize, - wdog: wdog, + URL: reqURL, + Resp: resp, + out: f, + completed: completed, + size: remoteSize, + wdog: wdog, + pollCB: config.PollFunction, + pollInterval: config.PollInterval, }, nil } diff --git a/downloader_config.go b/downloader_config.go index e6fce22..c75a556 100644 --- a/downloader_config.go +++ b/downloader_config.go @@ -30,6 +30,11 @@ type Config struct { // InactivityTimeout is the duration after which, if no data is received, // the download is aborted. If set to 0, no timeout is applied. InactivityTimeout time.Duration + // PollFunction is an optional function that will be called periodically + // during the download to report progress. + PollFunction func(current, size int64) + // PollInterval is the interval at which the PollFunction is called. + PollInterval time.Duration } var defaultConfig Config = Config{} diff --git a/downloader_test.go b/downloader_test.go index c2baaa7..c2a08c7 100644 --- a/downloader_test.go +++ b/downloader_test.go @@ -137,16 +137,20 @@ func TestInvalidRequest(t *testing.T) { func TestRunAndPool(t *testing.T) { tmpFile := makeTmpFile(t) - d, err := downloader.Download(tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2") - require.NoError(t, err) prevCurr := int64(0) callCount := 0 - callback := func(curr int64) { + callback := func(curr, size int64) { require.True(t, prevCurr <= curr) prevCurr = curr callCount++ } - require.NoError(t, d.RunAndPoll(callback, time.Millisecond)) + + config := downloader.GetDefaultConfig() + config.PollFunction = callback + config.PollInterval = time.Millisecond + d, err := downloader.DownloadWithConfig(tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2", config) + require.NoError(t, err) + require.NoError(t, d.Run()) fmt.Printf("callback called %d times\n", callCount) require.Greater(t, callCount, 10) require.Equal(t, int64(4897949), d.Completed()) @@ -216,23 +220,28 @@ func TestContextCancelation(t *testing.T) { tmpFile := makeTmpFile(t) + max := int64(0) + startTime := time.Now() + callback := func(curr, size int64) { + fmt.Println("Downloaded", curr, "bytes of", size, ". Elapsed:", time.Since(startTime)) + max = curr + } + ctx, cancel := context.WithCancel(context.Background()) - d, err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, server.URL+"/slow", downloader.Config{}) + d, err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, server.URL+"/slow", downloader.Config{ + PollInterval: 100 * time.Millisecond, + PollFunction: callback, + }) require.NoError(t, err) // Cancel in two seconds - startTime := time.Now() go func() { time.Sleep(2 * time.Second) cancel() }() // Run slow download - max := int64(0) - err = d.RunAndPoll(func(curr int64) { - fmt.Println("Downloaded", curr, "bytes. Elapsed:", time.Since(startTime)) - max = curr - }, 100*time.Millisecond) + err = d.Run() require.EqualError(t, err, "context canceled") require.True(t, max < 210) fmt.Println("Context canceled successfully") diff --git a/examples/download_file/main.go b/examples/download_file/main.go index a33629c..e6c0f3d 100644 --- a/examples/download_file/main.go +++ b/examples/download_file/main.go @@ -23,15 +23,17 @@ func main() { } defer os.RemoveAll(tmp) - d, err := downloader.Download(filepath.Join(tmp, "test.txt"), "https://go.bug.st/test.txt") + d, err := downloader.DownloadWithConfig(filepath.Join(tmp, "test.txt"), "https://go.bug.st/test.txt", downloader.Config{ + PollInterval: time.Millisecond, + PollFunction: func(current, size int64) { + fmt.Printf("Downloaded %d / %d bytes (%.2f%%)\n", current, size, float64(current)*100.0/float64(size)) + }, + }) if err != nil { log.Fatal(err) } - progressCB := func(current int64) { - fmt.Printf("Downloaded %d / %d bytes (%.2f%%)\n", current, d.Size(), float64(current)*100.0/float64(d.Size())) - } - if err := d.RunAndPoll(progressCB, time.Millisecond); err != nil { + if err := d.Run(); err != nil { log.Fatal(err) } } From 32cdb629bba53b2d26d355b65ebbd95a62fb3a91 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Sun, 11 Jan 2026 23:28:51 +0100 Subject: [PATCH 2/7] Remove the Downloader object and inline the Run method --- downloader.go | 202 ++++++++++----------------- downloader_test.go | 172 ++++++++++++----------- examples/check_download_size/main.go | 5 +- examples/download_file/main.go | 9 +- 4 files changed, 168 insertions(+), 220 deletions(-) diff --git a/downloader.go b/downloader.go index 07c19f6..64e3da3 100644 --- a/downloader.go +++ b/downloader.go @@ -16,113 +16,10 @@ import ( "time" ) -// Downloader is an asynchronous downloader -type Downloader struct { - URL string - Resp *http.Response - out *os.File - completed int64 - completedLock sync.Mutex - size int64 - err error - wdog watchdog - pollCB func(current, size int64) - pollInterval time.Duration -} - -// Size return the size of the download (or -1 if the server doesn't provide it) -func (d *Downloader) Size() int64 { - return d.size -} - -// Run starts the downloader and waits until it completes the download. -func (d *Downloader) Run() error { - if d.pollCB == nil { - // Run the non-interactive download - return d.run() - } - - t := time.NewTicker(d.pollInterval) - defer t.Stop() - - res := make(chan error, 1) - go func() { - res <- d.run() - }() - for { - select { - case <-t.C: - d.pollCB(d.Completed(), d.size) - case err := <-res: - d.pollCB(d.Completed(), d.size) - return err - } - } -} - -func (d *Downloader) run() error { - d.completedLock.Lock() - skip := (d.completed == d.size) - d.completedLock.Unlock() - if skip { - return d.Error() - } - - defer func() { - d.out.Close() - d.Resp.Body.Close() - d.wdog.Cancel() - }() - - in := d.Resp.Body - buff := [4096]byte{} - for { - n, readErr := in.Read(buff[:]) - if n > 0 { - if _, writeErr := d.out.Write(buff[:n]); writeErr != nil { - // Error writing to file - d.err = writeErr - break - } - - // Update completed bytes count - d.completedLock.Lock() - d.completed += int64(n) - d.completedLock.Unlock() - - // Extend inactivity timeout deadline - d.wdog.Kick() - } - if readErr == io.EOF { - // Download completed successfully! - break - } - if readErr != nil { - // Network or other error - d.err = readErr - break - } - } - return d.Error() -} - -// Error returns the error during download or nil if no errors happened -func (d *Downloader) Error() error { - return d.err -} - -// Completed returns the bytes read so far -func (d *Downloader) Completed() int64 { - d.completedLock.Lock() - res := d.completed - d.completedLock.Unlock() - return res -} - // Download returns an asynchronous downloader that will download the specified url // in the specified file. A download resume is tried if a file shorter than the requested // url is already present. -func Download(file string, reqURL string) (*Downloader, error) { +func Download(file string, reqURL string) error { return DownloadWithConfig(file, reqURL, GetDefaultConfig()) } @@ -130,7 +27,7 @@ func Download(file string, reqURL string) (*Downloader, error) { // returns an asynchronous downloader that will download the specified url // in the specified file. A download resume is tried if a file shorter than the requested // url is already present. -func DownloadWithConfig(file string, reqURL string, config Config) (*Downloader, error) { +func DownloadWithConfig(file string, reqURL string, config Config) error { return DownloadWithConfigAndContext(context.Background(), file, reqURL, config) } @@ -165,7 +62,7 @@ func doHeadRequest(ctx context.Context, reqURL string, config Config) (*http.Res // A previous download is resumed if the local file is shorter than the remote file. // The download is skipped if the local file has the same size of the remote file. // The download is restarted from scratch if the local file is larger than the remote file. -func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config) (*Downloader, error) { +func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config) error { clientCanResume := !config.DoNotResumeDownload // Gather information about local file @@ -177,7 +74,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin // Perform a HEAD call to gather information about the server capabilities and remote file headResp, err := doHeadRequest(ctx, reqURL, config) if err != nil { - return nil, err + return err } remoteSize := headResp.ContentLength // -1 if server doesn't send Content-Length serverCanResume := (headResp.Header.Get("Accept-Ranges") == "bytes") && (remoteSize != -1) @@ -188,7 +85,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin acceptError = config.AcceptFunc(headResp) } if acceptError != nil { - return nil, acceptError + return acceptError } // If we are allowed to resume a download, check the local file size and decide how to proceed @@ -196,12 +93,10 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin if clientCanResume { if localSize == remoteSize { // Size matches: assume the file is already downloaded - return &Downloader{ - URL: reqURL, - Resp: headResp, - completed: remoteSize, - size: remoteSize, - }, nil + if config.PollFunction != nil { + config.PollFunction(remoteSize, remoteSize) + } + return nil } if localSize < remoteSize { // Local file is smaller than remote file: resume download @@ -216,7 +111,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { wdog.Cancel() - return nil, fmt.Errorf("setting up HTTP request: %s", err) + return fmt.Errorf("setting up HTTP request: %s", err) } if config.ExtraHeaders != nil { for k, v := range config.ExtraHeaders { @@ -230,7 +125,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin resp, err := config.HttpClient.Do(req) if err != nil { wdog.Cancel() - return nil, err + return err } // Check server response @@ -238,10 +133,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { _ = resp.Body.Close() wdog.Cancel() - return &Downloader{ - URL: reqURL, - Resp: resp, // Return response for further inspection - }, fmt.Errorf("server returned %s", resp.Status) + return fmt.Errorf("server returned %s", resp.Status) } } @@ -252,21 +144,67 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin } else { flags |= os.O_CREATE | os.O_TRUNC } - f, err := os.OpenFile(file, flags, 0644) + out, err := os.OpenFile(file, flags, 0644) if err != nil { _ = resp.Body.Close() wdog.Cancel() - return nil, fmt.Errorf("opening %s for writing: %s", file, err) + return fmt.Errorf("opening %s for writing: %s", file, err) } - return &Downloader{ - URL: reqURL, - Resp: resp, - out: f, - completed: completed, - size: remoteSize, - wdog: wdog, - pollCB: config.PollFunction, - pollInterval: config.PollInterval, - }, nil + var completedLock sync.Mutex + if config.PollFunction != nil { + update := func() { + completedLock.Lock() + _completed := completed + completedLock.Unlock() + config.PollFunction(_completed, remoteSize) + } + + // send initial update + update() + + var t *time.Timer + t = time.AfterFunc(config.PollInterval, func() { + // send intermediate updates + update() + t.Reset(config.PollInterval) + }) + + defer func() { + t.Stop() + // send final update + update() + }() + } + + defer func() { + out.Close() + resp.Body.Close() + wdog.Cancel() + }() + + in := resp.Body + buff := [4096]byte{} + for { + n, readErr := in.Read(buff[:]) + if n > 0 { + if _, writeErr := out.Write(buff[:n]); writeErr != nil { + // Error writing to file + return writeErr + } + + completedLock.Lock() + completed += int64(n) + completedLock.Unlock() + + // Extend inactivity timeout deadline + wdog.Kick() + } + if readErr == io.EOF { + return nil + } + if readErr != nil { + return readErr + } + } } diff --git a/downloader_test.go b/downloader_test.go index c2a08c7..a04a5f6 100644 --- a/downloader_test.go +++ b/downloader_test.go @@ -10,7 +10,6 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" "net/http/httptest" "os" @@ -37,13 +36,25 @@ func makeTmpFile(t *testing.T) string { func TestDownload(t *testing.T) { tmpFile := makeTmpFile(t) - d, err := downloader.Download(tmpFile, "https://go.bug.st/test.txt") + initialSize := int64(0) + fullSize := int64(8052) + finalSize := int64(0) + count := 0 + err := downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + PollInterval: time.Second, + PollFunction: func(current, size int64) { + if count == 0 { + initialSize = current + } + fullSize = size + finalSize = current + count++ + }, + }) require.NoError(t, err) - require.Equal(t, int64(0), d.Completed()) - require.Equal(t, int64(8052), d.Size()) - require.NoError(t, d.Run()) - require.Equal(t, int64(8052), d.Completed()) - require.Equal(t, int64(8052), d.Size()) + require.Equal(t, int64(0), initialSize) + require.Equal(t, int64(8052), fullSize) + require.Equal(t, int64(8052), finalSize) file1, err := os.ReadFile("testdata/test.txt") require.NoError(t, err) @@ -60,13 +71,25 @@ func TestResume(t *testing.T) { err = os.WriteFile(tmpFile, part, 0644) require.NoError(t, err) - d, err := downloader.Download(tmpFile, "https://go.bug.st/test.txt") - require.Equal(t, int64(3506), d.Completed()) - require.Equal(t, int64(8052), d.Size()) + initialSize := int64(0) + fullSize := int64(8052) + finalSize := int64(0) + count := 0 + err = downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + PollInterval: time.Second, + PollFunction: func(current, size int64) { + if count == 0 { + initialSize = current + } + fullSize = size + finalSize = current + count++ + }, + }) require.NoError(t, err) - require.NoError(t, d.Run()) - require.Equal(t, int64(8052), d.Completed()) - require.Equal(t, int64(8052), d.Size()) + require.Equal(t, int64(3506), initialSize) + require.Equal(t, int64(8052), fullSize) + require.Equal(t, int64(8052), finalSize) file1, err := os.ReadFile("testdata/test.txt") require.NoError(t, err) @@ -82,12 +105,26 @@ func TestResumeOnAlreadyCompletedFile(t *testing.T) { require.NoError(t, err) require.NoError(t, os.WriteFile(tmpFile, full, 0644)) - d, err := downloader.Download(tmpFile, "https://go.bug.st/test.txt") + initialSize := int64(0) + fullSize := int64(8052) + finalSize := int64(0) + count := 0 + err = downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + PollInterval: time.Second, + PollFunction: func(current, size int64) { + if count == 0 { + initialSize = current + } + fullSize = size + finalSize = current + count++ + }, + }) + require.NoError(t, err) + require.Equal(t, int64(8052), initialSize) + require.Equal(t, int64(8052), fullSize) + require.Equal(t, int64(8052), finalSize) require.NoError(t, err) - require.Equal(t, int64(8052), d.Completed()) - require.Equal(t, int64(8052), d.Size()) - require.NoError(t, d.Run()) - require.Equal(t, int64(8052), d.Completed()) // Check file content is unchanged file2, err := os.ReadFile(tmpFile) @@ -103,15 +140,26 @@ func TestNoResume(t *testing.T) { err = os.WriteFile(tmpFile, part, 0644) require.NoError(t, err) - d, err := downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + initialSize := int64(0) + fullSize := int64(8052) + finalSize := int64(0) + count := 0 + err = downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ DoNotResumeDownload: true, + PollInterval: time.Second, + PollFunction: func(current, size int64) { + if count == 0 { + initialSize = current + } + fullSize = size + finalSize = current + count++ + }, }) - require.Equal(t, int64(0), d.Completed()) - require.Equal(t, int64(8052), d.Size()) require.NoError(t, err) - require.NoError(t, d.Run()) - require.Equal(t, int64(8052), d.Completed()) - require.Equal(t, int64(8052), d.Size()) + require.Equal(t, int64(0), initialSize) + require.Equal(t, int64(8052), fullSize) + require.Equal(t, int64(8052), finalSize) file1, err := os.ReadFile("testdata/test.txt") require.NoError(t, err) @@ -123,14 +171,8 @@ func TestNoResume(t *testing.T) { func TestInvalidRequest(t *testing.T) { tmpFile := makeTmpFile(t) - d, err := downloader.Download(tmpFile, "asd://go.bug.st/test.txt") + err := downloader.Download(tmpFile, "asd://go.bug.st/test.txt") require.Error(t, err) - require.Nil(t, d) - fmt.Println("ERROR:", err) - - d, err = downloader.Download(tmpFile, "://") - require.Error(t, err) - require.Nil(t, d) fmt.Println("ERROR:", err) } @@ -148,52 +190,34 @@ func TestRunAndPool(t *testing.T) { config := downloader.GetDefaultConfig() config.PollFunction = callback config.PollInterval = time.Millisecond - d, err := downloader.DownloadWithConfig(tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2", config) + err := downloader.DownloadWithConfig(tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2", config) require.NoError(t, err) - require.NoError(t, d.Run()) fmt.Printf("callback called %d times\n", callCount) require.Greater(t, callCount, 10) - require.Equal(t, int64(4897949), d.Completed()) + require.Equal(t, int64(4897949), prevCurr) } func TestErrorOnFileOpening(t *testing.T) { unaccessibleFile := filepath.Join(os.TempDir(), "nonexistentdir", "test.txt") - d, err := downloader.Download(unaccessibleFile, "http://go.bug.st/test.txt") + err := downloader.Download(unaccessibleFile, "http://go.bug.st/test.txt") require.Error(t, err) - require.Nil(t, d) -} - -type roundTripper struct { - UserAgent string - transport http.Transport -} - -func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - req.Header["User-Agent"] = []string{r.UserAgent} - return r.transport.RoundTrip(req) } // TestApplyUserAgentHeaderUsingConfig test uses the https://postman-echo.com/ service func TestApplyUserAgentHeaderUsingConfig(t *testing.T) { - type echoBody struct { - Headers map[string]string - } - tmpFile := makeTmpFile(t) - defer os.Remove(tmpFile) - httpClient := http.Client{ - Transport: &roundTripper{UserAgent: "go-downloader / 0.0.0-test"}, - } - config := downloader.Config{ - HttpClient: httpClient, - } - - d, err := downloader.DownloadWithConfig(tmpFile, "https://postman-echo.com/headers", config) + err := downloader.DownloadWithConfig(tmpFile, "https://postman-echo.com/headers", downloader.Config{ + ExtraHeaders: map[string]string{ + "User-Agent": "go-downloader / 0.0.0-test", + }, + }) require.NoError(t, err) - testEchoBody := echoBody{} - body, err := io.ReadAll(d.Resp.Body) + testEchoBody := struct { + Headers map[string]string + }{} + body, err := os.ReadFile(tmpFile) require.NoError(t, err) err = json.Unmarshal(body, &testEchoBody) require.NoError(t, err) @@ -228,12 +252,6 @@ func TestContextCancelation(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) - d, err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, server.URL+"/slow", downloader.Config{ - PollInterval: 100 * time.Millisecond, - PollFunction: callback, - }) - require.NoError(t, err) - // Cancel in two seconds go func() { time.Sleep(2 * time.Second) @@ -241,7 +259,10 @@ func TestContextCancelation(t *testing.T) { }() // Run slow download - err = d.Run() + err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, server.URL+"/slow", downloader.Config{ + PollInterval: 100 * time.Millisecond, + PollFunction: callback, + }) require.EqualError(t, err, "context canceled") require.True(t, max < 210) fmt.Println("Context canceled successfully") @@ -264,9 +285,8 @@ func TestTimeoutOnHEADCall(t *testing.T) { } startTime := time.Now() - d, err := downloader.DownloadWithConfig(tmpFile, server.URL+"/timeout", config) + err := downloader.DownloadWithConfig(tmpFile, server.URL+"/timeout", config) require.Error(t, err) - require.Nil(t, d) require.Contains(t, err.Error(), "context deadline exceeded") elapsed := time.Since(startTime) require.True(t, elapsed < 5*time.Second) @@ -293,9 +313,8 @@ func TestTimeoutOnGETCall(t *testing.T) { } startTime := time.Now() - d, err := downloader.DownloadWithConfig(tmpFile, server.URL+"/slow", config) + err := downloader.DownloadWithConfig(tmpFile, server.URL+"/slow", config) require.Error(t, err) - require.Nil(t, d) require.Contains(t, err.Error(), "i/o timeout") elapsed := time.Since(startTime) require.True(t, elapsed < 5*time.Second) @@ -336,18 +355,15 @@ func TestInactivityTimeout(t *testing.T) { t.Cleanup(server.Close) // Request download with inactivity timeout of 500ms + // (should timeout due to inactivity) tmpFile := makeTmpFile(t) - d, err := downloader.DownloadWithConfigAndContext( + startTime := time.Now() + err := downloader.DownloadWithConfigAndContext( t.Context(), tmpFile, server.URL+"/inactive", downloader.Config{ InactivityTimeout: 500 * time.Millisecond, }, ) - require.NoError(t, err) - - // Run the download, should timeout due to inactivity - startTime := time.Now() - err = d.Run() elapsed := time.Since(startTime) // Check that we got a timeout error diff --git a/examples/check_download_size/main.go b/examples/check_download_size/main.go index 2690014..7c40465 100644 --- a/examples/check_download_size/main.go +++ b/examples/check_download_size/main.go @@ -15,15 +15,14 @@ import ( ) func main() { - _, err := downloader.DownloadWithConfig("test.txt", "https://go.bug.st/test.txt", downloader.Config{ + if err := downloader.DownloadWithConfig("test.txt", "https://go.bug.st/test.txt", downloader.Config{ AcceptFunc: func(head *http.Response) error { if head.ContentLength > 2000 { return fmt.Errorf("insufficient space for download") } return nil }, - }) - if err != nil { + }); err != nil { log.Fatal(err) } diff --git a/examples/download_file/main.go b/examples/download_file/main.go index e6c0f3d..7400226 100644 --- a/examples/download_file/main.go +++ b/examples/download_file/main.go @@ -23,17 +23,12 @@ func main() { } defer os.RemoveAll(tmp) - d, err := downloader.DownloadWithConfig(filepath.Join(tmp, "test.txt"), "https://go.bug.st/test.txt", downloader.Config{ + if err := downloader.DownloadWithConfig(filepath.Join(tmp, "test.txt"), "https://go.bug.st/test.txt", downloader.Config{ PollInterval: time.Millisecond, PollFunction: func(current, size int64) { fmt.Printf("Downloaded %d / %d bytes (%.2f%%)\n", current, size, float64(current)*100.0/float64(size)) }, - }) - if err != nil { - log.Fatal(err) - } - - if err := d.Run(); err != nil { + }); err != nil { log.Fatal(err) } } From 8b01d7ea2751359273172c4f1def5852dfd552d5 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 12 Jan 2026 01:36:06 +0100 Subject: [PATCH 3/7] Removed redundant function DownloadWithConfigAndContext --- downloader.go | 34 +++++++++++----------------- downloader_test.go | 24 ++++++++++---------- examples/check_download_size/main.go | 3 ++- examples/download_file/main.go | 17 +++++++++----- 4 files changed, 38 insertions(+), 40 deletions(-) diff --git a/downloader.go b/downloader.go index 64e3da3..deec3e7 100644 --- a/downloader.go +++ b/downloader.go @@ -16,21 +16,6 @@ import ( "time" ) -// Download returns an asynchronous downloader that will download the specified url -// in the specified file. A download resume is tried if a file shorter than the requested -// url is already present. -func Download(file string, reqURL string) error { - return DownloadWithConfig(file, reqURL, GetDefaultConfig()) -} - -// DownloadWithConfig applies an additional configuration to the http client and -// returns an asynchronous downloader that will download the specified url -// in the specified file. A download resume is tried if a file shorter than the requested -// url is already present. -func DownloadWithConfig(file string, reqURL string, config Config) error { - return DownloadWithConfigAndContext(context.Background(), file, reqURL, config) -} - func doHeadRequest(ctx context.Context, reqURL string, config Config) (*http.Response, error) { headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil) if err != nil { @@ -56,13 +41,20 @@ func doHeadRequest(ctx context.Context, reqURL string, config Config) (*http.Res return headResp, nil } -// DownloadWithConfigAndContext applies an additional configuration to the http client and -// returns an asynchronous downloader that will download the specified url -// in the specified file. -// A previous download is resumed if the local file is shorter than the remote file. +// Download downloads the specified url in the specified file. +// A download resume is tried if a file shorter than the requested url is already present. +func Download(ctx context.Context, file string, reqURL string) error { + return DownloadWithConfig(ctx, file, reqURL, GetDefaultConfig()) +} + +// DownloadWithConfig applies an additional configuration to the http client and +// downloads download the specified url in the specified file. +// A previous download is resumed if the local file is shorter than the remote file, +// unless otherwise specified. // The download is skipped if the local file has the same size of the remote file. -// The download is restarted from scratch if the local file is larger than the remote file. -func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config) error { +// The download is restarted from scratch if the local file is larger than the remote +// file or if the server doesn't support resuming. +func DownloadWithConfig(ctx context.Context, file string, reqURL string, config Config) error { clientCanResume := !config.DoNotResumeDownload // Gather information about local file diff --git a/downloader_test.go b/downloader_test.go index a04a5f6..c44761f 100644 --- a/downloader_test.go +++ b/downloader_test.go @@ -40,7 +40,7 @@ func TestDownload(t *testing.T) { fullSize := int64(8052) finalSize := int64(0) count := 0 - err := downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + err := downloader.DownloadWithConfig(t.Context(), tmpFile, "https://go.bug.st/test.txt", downloader.Config{ PollInterval: time.Second, PollFunction: func(current, size int64) { if count == 0 { @@ -75,7 +75,7 @@ func TestResume(t *testing.T) { fullSize := int64(8052) finalSize := int64(0) count := 0 - err = downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + err = downloader.DownloadWithConfig(t.Context(), tmpFile, "https://go.bug.st/test.txt", downloader.Config{ PollInterval: time.Second, PollFunction: func(current, size int64) { if count == 0 { @@ -109,7 +109,7 @@ func TestResumeOnAlreadyCompletedFile(t *testing.T) { fullSize := int64(8052) finalSize := int64(0) count := 0 - err = downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + err = downloader.DownloadWithConfig(t.Context(), tmpFile, "https://go.bug.st/test.txt", downloader.Config{ PollInterval: time.Second, PollFunction: func(current, size int64) { if count == 0 { @@ -144,7 +144,7 @@ func TestNoResume(t *testing.T) { fullSize := int64(8052) finalSize := int64(0) count := 0 - err = downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + err = downloader.DownloadWithConfig(t.Context(), tmpFile, "https://go.bug.st/test.txt", downloader.Config{ DoNotResumeDownload: true, PollInterval: time.Second, PollFunction: func(current, size int64) { @@ -171,7 +171,7 @@ func TestNoResume(t *testing.T) { func TestInvalidRequest(t *testing.T) { tmpFile := makeTmpFile(t) - err := downloader.Download(tmpFile, "asd://go.bug.st/test.txt") + err := downloader.Download(t.Context(), tmpFile, "asd://go.bug.st/test.txt") require.Error(t, err) fmt.Println("ERROR:", err) } @@ -190,7 +190,7 @@ func TestRunAndPool(t *testing.T) { config := downloader.GetDefaultConfig() config.PollFunction = callback config.PollInterval = time.Millisecond - err := downloader.DownloadWithConfig(tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2", config) + err := downloader.DownloadWithConfig(t.Context(), tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2", config) require.NoError(t, err) fmt.Printf("callback called %d times\n", callCount) require.Greater(t, callCount, 10) @@ -199,7 +199,7 @@ func TestRunAndPool(t *testing.T) { func TestErrorOnFileOpening(t *testing.T) { unaccessibleFile := filepath.Join(os.TempDir(), "nonexistentdir", "test.txt") - err := downloader.Download(unaccessibleFile, "http://go.bug.st/test.txt") + err := downloader.Download(t.Context(), unaccessibleFile, "http://go.bug.st/test.txt") require.Error(t, err) } @@ -207,7 +207,7 @@ func TestErrorOnFileOpening(t *testing.T) { func TestApplyUserAgentHeaderUsingConfig(t *testing.T) { tmpFile := makeTmpFile(t) - err := downloader.DownloadWithConfig(tmpFile, "https://postman-echo.com/headers", downloader.Config{ + err := downloader.DownloadWithConfig(t.Context(), tmpFile, "https://postman-echo.com/headers", downloader.Config{ ExtraHeaders: map[string]string{ "User-Agent": "go-downloader / 0.0.0-test", }, @@ -259,7 +259,7 @@ func TestContextCancelation(t *testing.T) { }() // Run slow download - err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, server.URL+"/slow", downloader.Config{ + err := downloader.DownloadWithConfig(ctx, tmpFile, server.URL+"/slow", downloader.Config{ PollInterval: 100 * time.Millisecond, PollFunction: callback, }) @@ -285,7 +285,7 @@ func TestTimeoutOnHEADCall(t *testing.T) { } startTime := time.Now() - err := downloader.DownloadWithConfig(tmpFile, server.URL+"/timeout", config) + err := downloader.DownloadWithConfig(t.Context(), tmpFile, server.URL+"/timeout", config) require.Error(t, err) require.Contains(t, err.Error(), "context deadline exceeded") elapsed := time.Since(startTime) @@ -313,7 +313,7 @@ func TestTimeoutOnGETCall(t *testing.T) { } startTime := time.Now() - err := downloader.DownloadWithConfig(tmpFile, server.URL+"/slow", config) + err := downloader.DownloadWithConfig(t.Context(), tmpFile, server.URL+"/slow", config) require.Error(t, err) require.Contains(t, err.Error(), "i/o timeout") elapsed := time.Since(startTime) @@ -358,7 +358,7 @@ func TestInactivityTimeout(t *testing.T) { // (should timeout due to inactivity) tmpFile := makeTmpFile(t) startTime := time.Now() - err := downloader.DownloadWithConfigAndContext( + err := downloader.DownloadWithConfig( t.Context(), tmpFile, server.URL+"/inactive", downloader.Config{ InactivityTimeout: 500 * time.Millisecond, diff --git a/examples/check_download_size/main.go b/examples/check_download_size/main.go index 7c40465..a2c3141 100644 --- a/examples/check_download_size/main.go +++ b/examples/check_download_size/main.go @@ -7,6 +7,7 @@ package main import ( + "context" "fmt" "log" "net/http" @@ -15,7 +16,7 @@ import ( ) func main() { - if err := downloader.DownloadWithConfig("test.txt", "https://go.bug.st/test.txt", downloader.Config{ + if err := downloader.DownloadWithConfig(context.Background(), "test.txt", "https://go.bug.st/test.txt", downloader.Config{ AcceptFunc: func(head *http.Response) error { if head.ContentLength > 2000 { return fmt.Errorf("insufficient space for download") diff --git a/examples/download_file/main.go b/examples/download_file/main.go index 7400226..3a93298 100644 --- a/examples/download_file/main.go +++ b/examples/download_file/main.go @@ -7,6 +7,7 @@ package main import ( + "context" "fmt" "log" "os" @@ -23,12 +24,16 @@ func main() { } defer os.RemoveAll(tmp) - if err := downloader.DownloadWithConfig(filepath.Join(tmp, "test.txt"), "https://go.bug.st/test.txt", downloader.Config{ - PollInterval: time.Millisecond, - PollFunction: func(current, size int64) { - fmt.Printf("Downloaded %d / %d bytes (%.2f%%)\n", current, size, float64(current)*100.0/float64(size)) - }, - }); err != nil { + if err := downloader.DownloadWithConfig( + context.Background(), + filepath.Join(tmp, "test.txt"), + "https://go.bug.st/test.txt", + downloader.Config{ + PollInterval: time.Millisecond, + PollFunction: func(current, size int64) { + fmt.Printf("Downloaded %d / %d bytes (%.2f%%)\n", current, size, float64(current)*100.0/float64(size)) + }, + }); err != nil { log.Fatal(err) } } From 4ed15a0bcf4325b7fb922ed447d6d968a3a1cfb0 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 12 Jan 2026 01:42:46 +0100 Subject: [PATCH 4/7] Defer early and remove manual closes --- downloader.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/downloader.go b/downloader.go index deec3e7..3149da3 100644 --- a/downloader.go +++ b/downloader.go @@ -100,9 +100,9 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config // Perform the actual GET request // Setup inactivity timeout for the GET request ctx, wdog := newWatchdog(ctx, config.InactivityTimeout) + defer wdog.Cancel() req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { - wdog.Cancel() return fmt.Errorf("setting up HTTP request: %s", err) } if config.ExtraHeaders != nil { @@ -116,15 +116,13 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config } resp, err := config.HttpClient.Do(req) if err != nil { - wdog.Cancel() return err } + defer resp.Body.Close() // Check server response if !config.DoNotErrorOnNon2xxStatusCode { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - _ = resp.Body.Close() - wdog.Cancel() return fmt.Errorf("server returned %s", resp.Status) } } @@ -138,10 +136,9 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config } out, err := os.OpenFile(file, flags, 0644) if err != nil { - _ = resp.Body.Close() - wdog.Cancel() return fmt.Errorf("opening %s for writing: %s", file, err) } + defer out.Close() var completedLock sync.Mutex if config.PollFunction != nil { @@ -169,12 +166,6 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config }() } - defer func() { - out.Close() - resp.Body.Close() - wdog.Cancel() - }() - in := resp.Body buff := [4096]byte{} for { From bfeed6f4c854987813f7ad57d803059f2eaa5cce Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 12 Jan 2026 01:45:41 +0100 Subject: [PATCH 5/7] Add a default value for PollInterval --- downloader.go | 7 +++++-- downloader_config.go | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/downloader.go b/downloader.go index 3149da3..15d11bd 100644 --- a/downloader.go +++ b/downloader.go @@ -7,6 +7,7 @@ package downloader import ( + "cmp" "context" "fmt" "io" @@ -140,6 +141,7 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config } defer out.Close() + // Start polling goroutine if requested var completedLock sync.Mutex if config.PollFunction != nil { update := func() { @@ -152,11 +154,12 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config // send initial update update() + interval := cmp.Or(config.PollInterval.Abs(), 250*time.Millisecond) var t *time.Timer - t = time.AfterFunc(config.PollInterval, func() { + t = time.AfterFunc(interval, func() { // send intermediate updates update() - t.Reset(config.PollInterval) + t.Reset(interval) }) defer func() { diff --git a/downloader_config.go b/downloader_config.go index c75a556..1885454 100644 --- a/downloader_config.go +++ b/downloader_config.go @@ -33,7 +33,7 @@ type Config struct { // PollFunction is an optional function that will be called periodically // during the download to report progress. PollFunction func(current, size int64) - // PollInterval is the interval at which the PollFunction is called. + // PollInterval is the interval at which the PollFunction is called (default is 250ms if not specified). PollInterval time.Duration } From 3b065a85f640991455985109d8270729536df23d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 12 Jan 2026 10:48:03 +0100 Subject: [PATCH 6/7] Use atomic instead of locks --- downloader.go | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/downloader.go b/downloader.go index 15d11bd..bd757eb 100644 --- a/downloader.go +++ b/downloader.go @@ -13,7 +13,7 @@ import ( "io" "net/http" "os" - "sync" + "sync/atomic" "time" ) @@ -82,7 +82,6 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config } // If we are allowed to resume a download, check the local file size and decide how to proceed - var completed int64 if clientCanResume { if localSize == remoteSize { // Size matches: assume the file is already downloaded @@ -91,10 +90,9 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config } return nil } - if localSize < remoteSize { - // Local file is smaller than remote file: resume download - // Remote size is unknown: resume download anyway - completed = localSize + if localSize > remoteSize { + // Local file is larger than remote file: restart download + clientCanResume = false } } @@ -111,9 +109,9 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config req.Header.Set(k, v) } } - resumeDownload := clientCanResume && serverCanResume && completed > 0 + resumeDownload := clientCanResume && serverCanResume && localSize > 0 if resumeDownload { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", completed)) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", localSize)) } resp, err := config.HttpClient.Do(req) if err != nil { @@ -142,13 +140,13 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config defer out.Close() // Start polling goroutine if requested - var completedLock sync.Mutex + var completed atomic.Int64 + if resumeDownload { + completed.Store(localSize) + } if config.PollFunction != nil { update := func() { - completedLock.Lock() - _completed := completed - completedLock.Unlock() - config.PollFunction(_completed, remoteSize) + config.PollFunction(completed.Load(), remoteSize) } // send initial update @@ -179,9 +177,7 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config return writeErr } - completedLock.Lock() - completed += int64(n) - completedLock.Unlock() + completed.Add(int64(n)) // Extend inactivity timeout deadline wdog.Kick() From 950e0f678f951d5e663a6eec0dec75259152570b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 12 Jan 2026 10:58:56 +0100 Subject: [PATCH 7/7] Fixed race condition --- downloader.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/downloader.go b/downloader.go index bd757eb..e89063c 100644 --- a/downloader.go +++ b/downloader.go @@ -13,6 +13,7 @@ import ( "io" "net/http" "os" + "sync" "sync/atomic" "time" ) @@ -145,7 +146,10 @@ func DownloadWithConfig(ctx context.Context, file string, reqURL string, config completed.Store(localSize) } if config.PollFunction != nil { + var updateLock sync.Mutex update := func() { + updateLock.Lock() + defer updateLock.Unlock() config.PollFunction(completed.Load(), remoteSize) }