diff --git a/downloader.go b/downloader.go index bed25be..e89063c 100644 --- a/downloader.go +++ b/downloader.go @@ -7,130 +7,17 @@ package downloader import ( + "cmp" "context" "fmt" "io" "net/http" "os" "sync" + "sync/atomic" "time" ) -// Downloader is an asynchronous downloader -type Downloader struct { - URL string - Resp *http.Response - out *os.File - completed int64 - completedLock sync.Mutex - size int64 - err error - wdog watchdog -} - -// Size return the size of the download (or -1 if the server doesn't provide it) -func (d *Downloader) Size() int64 { - return d.size -} - -// RunAndPoll starts the downloader copy-loop and calls the poll function every -// interval time to update progress. -func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration) error { - t := time.NewTicker(interval) - defer t.Stop() - - res := make(chan error, 1) - go func() { - res <- d.Run() - }() - for { - select { - case <-t.C: - poll(d.Completed()) - case err := <-res: - poll(d.Completed()) - return err - } - } -} - -// Run starts the downloader and waits until it completes the download. -// This method can be run in a goroutine to perform an asynchronous download; -// it will close the Done channel when the download is completed or an error occurs. -func (d *Downloader) Run() error { - d.completedLock.Lock() - skip := (d.completed == d.size) - d.completedLock.Unlock() - if skip { - return d.Error() - } - - defer func() { - d.out.Close() - d.Resp.Body.Close() - d.wdog.Cancel() - }() - - in := d.Resp.Body - buff := [4096]byte{} - for { - n, readErr := in.Read(buff[:]) - if n > 0 { - if _, writeErr := d.out.Write(buff[:n]); writeErr != nil { - // Error writing to file - d.err = writeErr - break - } - - // Update completed bytes count - d.completedLock.Lock() - d.completed += int64(n) - d.completedLock.Unlock() - - // Extend inactivity timeout deadline - d.wdog.Kick() - } - if readErr == io.EOF { - // Download completed successfully! - break - } - if readErr != nil { - // Network or other error - d.err = readErr - break - } - } - return d.Error() -} - -// Error returns the error during download or nil if no errors happened -func (d *Downloader) Error() error { - return d.err -} - -// Completed returns the bytes read so far -func (d *Downloader) Completed() int64 { - d.completedLock.Lock() - res := d.completed - d.completedLock.Unlock() - return res -} - -// Download returns an asynchronous downloader that will download the specified url -// in the specified file. A download resume is tried if a file shorter than the requested -// url is already present. -func Download(file string, reqURL string) (*Downloader, error) { - return DownloadWithConfig(file, reqURL, GetDefaultConfig()) -} - -// DownloadWithConfig applies an additional configuration to the http client and -// returns an asynchronous downloader that will download the specified url -// in the specified file. A download resume is tried if a file shorter than the requested -// url is already present. -func DownloadWithConfig(file string, reqURL string, config Config) (*Downloader, error) { - return DownloadWithConfigAndContext(context.Background(), file, reqURL, config) -} - func doHeadRequest(ctx context.Context, reqURL string, config Config) (*http.Response, error) { headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil) if err != nil { @@ -156,13 +43,20 @@ func doHeadRequest(ctx context.Context, reqURL string, config Config) (*http.Res return headResp, nil } -// DownloadWithConfigAndContext applies an additional configuration to the http client and -// returns an asynchronous downloader that will download the specified url -// in the specified file. -// A previous download is resumed if the local file is shorter than the remote file. +// Download downloads the specified url in the specified file. +// A download resume is tried if a file shorter than the requested url is already present. +func Download(ctx context.Context, file string, reqURL string) error { + return DownloadWithConfig(ctx, file, reqURL, GetDefaultConfig()) +} + +// DownloadWithConfig applies an additional configuration to the http client and +// downloads download the specified url in the specified file. +// A previous download is resumed if the local file is shorter than the remote file, +// unless otherwise specified. // The download is skipped if the local file has the same size of the remote file. -// The download is restarted from scratch if the local file is larger than the remote file. -func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config) (*Downloader, error) { +// The download is restarted from scratch if the local file is larger than the remote +// file or if the server doesn't support resuming. +func DownloadWithConfig(ctx context.Context, file string, reqURL string, config Config) error { clientCanResume := !config.DoNotResumeDownload // Gather information about local file @@ -174,7 +68,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin // Perform a HEAD call to gather information about the server capabilities and remote file headResp, err := doHeadRequest(ctx, reqURL, config) if err != nil { - return nil, err + return err } remoteSize := headResp.ContentLength // -1 if server doesn't send Content-Length serverCanResume := (headResp.Header.Get("Accept-Ranges") == "bytes") && (remoteSize != -1) @@ -185,60 +79,51 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin acceptError = config.AcceptFunc(headResp) } if acceptError != nil { - return nil, acceptError + return acceptError } // If we are allowed to resume a download, check the local file size and decide how to proceed - var completed int64 if clientCanResume { if localSize == remoteSize { // Size matches: assume the file is already downloaded - return &Downloader{ - URL: reqURL, - Resp: headResp, - completed: remoteSize, - size: remoteSize, - }, nil + if config.PollFunction != nil { + config.PollFunction(remoteSize, remoteSize) + } + return nil } - if localSize < remoteSize { - // Local file is smaller than remote file: resume download - // Remote size is unknown: resume download anyway - completed = localSize + if localSize > remoteSize { + // Local file is larger than remote file: restart download + clientCanResume = false } } // Perform the actual GET request // Setup inactivity timeout for the GET request ctx, wdog := newWatchdog(ctx, config.InactivityTimeout) + defer wdog.Cancel() req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { - wdog.Cancel() - return nil, fmt.Errorf("setting up HTTP request: %s", err) + return fmt.Errorf("setting up HTTP request: %s", err) } if config.ExtraHeaders != nil { for k, v := range config.ExtraHeaders { req.Header.Set(k, v) } } - resumeDownload := clientCanResume && serverCanResume && completed > 0 + resumeDownload := clientCanResume && serverCanResume && localSize > 0 if resumeDownload { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", completed)) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", localSize)) } resp, err := config.HttpClient.Do(req) if err != nil { - wdog.Cancel() - return nil, err + return err } + defer resp.Body.Close() // Check server response if !config.DoNotErrorOnNon2xxStatusCode { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - _ = resp.Body.Close() - wdog.Cancel() - return &Downloader{ - URL: reqURL, - Resp: resp, // Return response for further inspection - }, fmt.Errorf("server returned %s", resp.Status) + return fmt.Errorf("server returned %s", resp.Status) } } @@ -249,19 +134,63 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin } else { flags |= os.O_CREATE | os.O_TRUNC } - f, err := os.OpenFile(file, flags, 0644) + out, err := os.OpenFile(file, flags, 0644) if err != nil { - _ = resp.Body.Close() - wdog.Cancel() - return nil, fmt.Errorf("opening %s for writing: %s", file, err) + return fmt.Errorf("opening %s for writing: %s", file, err) } + defer out.Close() + + // Start polling goroutine if requested + var completed atomic.Int64 + if resumeDownload { + completed.Store(localSize) + } + if config.PollFunction != nil { + var updateLock sync.Mutex + update := func() { + updateLock.Lock() + defer updateLock.Unlock() + config.PollFunction(completed.Load(), remoteSize) + } + + // send initial update + update() - return &Downloader{ - URL: reqURL, - Resp: resp, - out: f, - completed: completed, - size: remoteSize, - wdog: wdog, - }, nil + interval := cmp.Or(config.PollInterval.Abs(), 250*time.Millisecond) + var t *time.Timer + t = time.AfterFunc(interval, func() { + // send intermediate updates + update() + t.Reset(interval) + }) + + defer func() { + t.Stop() + // send final update + update() + }() + } + + in := resp.Body + buff := [4096]byte{} + for { + n, readErr := in.Read(buff[:]) + if n > 0 { + if _, writeErr := out.Write(buff[:n]); writeErr != nil { + // Error writing to file + return writeErr + } + + completed.Add(int64(n)) + + // Extend inactivity timeout deadline + wdog.Kick() + } + if readErr == io.EOF { + return nil + } + if readErr != nil { + return readErr + } + } } diff --git a/downloader_config.go b/downloader_config.go index e6fce22..1885454 100644 --- a/downloader_config.go +++ b/downloader_config.go @@ -30,6 +30,11 @@ type Config struct { // InactivityTimeout is the duration after which, if no data is received, // the download is aborted. If set to 0, no timeout is applied. InactivityTimeout time.Duration + // PollFunction is an optional function that will be called periodically + // during the download to report progress. + PollFunction func(current, size int64) + // PollInterval is the interval at which the PollFunction is called (default is 250ms if not specified). + PollInterval time.Duration } var defaultConfig Config = Config{} diff --git a/downloader_test.go b/downloader_test.go index c2baaa7..c44761f 100644 --- a/downloader_test.go +++ b/downloader_test.go @@ -10,7 +10,6 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" "net/http/httptest" "os" @@ -37,13 +36,25 @@ func makeTmpFile(t *testing.T) string { func TestDownload(t *testing.T) { tmpFile := makeTmpFile(t) - d, err := downloader.Download(tmpFile, "https://go.bug.st/test.txt") + initialSize := int64(0) + fullSize := int64(8052) + finalSize := int64(0) + count := 0 + err := downloader.DownloadWithConfig(t.Context(), tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + PollInterval: time.Second, + PollFunction: func(current, size int64) { + if count == 0 { + initialSize = current + } + fullSize = size + finalSize = current + count++ + }, + }) require.NoError(t, err) - require.Equal(t, int64(0), d.Completed()) - require.Equal(t, int64(8052), d.Size()) - require.NoError(t, d.Run()) - require.Equal(t, int64(8052), d.Completed()) - require.Equal(t, int64(8052), d.Size()) + require.Equal(t, int64(0), initialSize) + require.Equal(t, int64(8052), fullSize) + require.Equal(t, int64(8052), finalSize) file1, err := os.ReadFile("testdata/test.txt") require.NoError(t, err) @@ -60,13 +71,25 @@ func TestResume(t *testing.T) { err = os.WriteFile(tmpFile, part, 0644) require.NoError(t, err) - d, err := downloader.Download(tmpFile, "https://go.bug.st/test.txt") - require.Equal(t, int64(3506), d.Completed()) - require.Equal(t, int64(8052), d.Size()) + initialSize := int64(0) + fullSize := int64(8052) + finalSize := int64(0) + count := 0 + err = downloader.DownloadWithConfig(t.Context(), tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + PollInterval: time.Second, + PollFunction: func(current, size int64) { + if count == 0 { + initialSize = current + } + fullSize = size + finalSize = current + count++ + }, + }) require.NoError(t, err) - require.NoError(t, d.Run()) - require.Equal(t, int64(8052), d.Completed()) - require.Equal(t, int64(8052), d.Size()) + require.Equal(t, int64(3506), initialSize) + require.Equal(t, int64(8052), fullSize) + require.Equal(t, int64(8052), finalSize) file1, err := os.ReadFile("testdata/test.txt") require.NoError(t, err) @@ -82,12 +105,26 @@ func TestResumeOnAlreadyCompletedFile(t *testing.T) { require.NoError(t, err) require.NoError(t, os.WriteFile(tmpFile, full, 0644)) - d, err := downloader.Download(tmpFile, "https://go.bug.st/test.txt") + initialSize := int64(0) + fullSize := int64(8052) + finalSize := int64(0) + count := 0 + err = downloader.DownloadWithConfig(t.Context(), tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + PollInterval: time.Second, + PollFunction: func(current, size int64) { + if count == 0 { + initialSize = current + } + fullSize = size + finalSize = current + count++ + }, + }) + require.NoError(t, err) + require.Equal(t, int64(8052), initialSize) + require.Equal(t, int64(8052), fullSize) + require.Equal(t, int64(8052), finalSize) require.NoError(t, err) - require.Equal(t, int64(8052), d.Completed()) - require.Equal(t, int64(8052), d.Size()) - require.NoError(t, d.Run()) - require.Equal(t, int64(8052), d.Completed()) // Check file content is unchanged file2, err := os.ReadFile(tmpFile) @@ -103,15 +140,26 @@ func TestNoResume(t *testing.T) { err = os.WriteFile(tmpFile, part, 0644) require.NoError(t, err) - d, err := downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + initialSize := int64(0) + fullSize := int64(8052) + finalSize := int64(0) + count := 0 + err = downloader.DownloadWithConfig(t.Context(), tmpFile, "https://go.bug.st/test.txt", downloader.Config{ DoNotResumeDownload: true, + PollInterval: time.Second, + PollFunction: func(current, size int64) { + if count == 0 { + initialSize = current + } + fullSize = size + finalSize = current + count++ + }, }) - require.Equal(t, int64(0), d.Completed()) - require.Equal(t, int64(8052), d.Size()) require.NoError(t, err) - require.NoError(t, d.Run()) - require.Equal(t, int64(8052), d.Completed()) - require.Equal(t, int64(8052), d.Size()) + require.Equal(t, int64(0), initialSize) + require.Equal(t, int64(8052), fullSize) + require.Equal(t, int64(8052), finalSize) file1, err := os.ReadFile("testdata/test.txt") require.NoError(t, err) @@ -123,73 +171,53 @@ func TestNoResume(t *testing.T) { func TestInvalidRequest(t *testing.T) { tmpFile := makeTmpFile(t) - d, err := downloader.Download(tmpFile, "asd://go.bug.st/test.txt") - require.Error(t, err) - require.Nil(t, d) - fmt.Println("ERROR:", err) - - d, err = downloader.Download(tmpFile, "://") + err := downloader.Download(t.Context(), tmpFile, "asd://go.bug.st/test.txt") require.Error(t, err) - require.Nil(t, d) fmt.Println("ERROR:", err) } func TestRunAndPool(t *testing.T) { tmpFile := makeTmpFile(t) - d, err := downloader.Download(tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2") - require.NoError(t, err) prevCurr := int64(0) callCount := 0 - callback := func(curr int64) { + callback := func(curr, size int64) { require.True(t, prevCurr <= curr) prevCurr = curr callCount++ } - require.NoError(t, d.RunAndPoll(callback, time.Millisecond)) + + config := downloader.GetDefaultConfig() + config.PollFunction = callback + config.PollInterval = time.Millisecond + err := downloader.DownloadWithConfig(t.Context(), tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2", config) + require.NoError(t, err) fmt.Printf("callback called %d times\n", callCount) require.Greater(t, callCount, 10) - require.Equal(t, int64(4897949), d.Completed()) + require.Equal(t, int64(4897949), prevCurr) } func TestErrorOnFileOpening(t *testing.T) { unaccessibleFile := filepath.Join(os.TempDir(), "nonexistentdir", "test.txt") - d, err := downloader.Download(unaccessibleFile, "http://go.bug.st/test.txt") + err := downloader.Download(t.Context(), unaccessibleFile, "http://go.bug.st/test.txt") require.Error(t, err) - require.Nil(t, d) -} - -type roundTripper struct { - UserAgent string - transport http.Transport -} - -func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - req.Header["User-Agent"] = []string{r.UserAgent} - return r.transport.RoundTrip(req) } // TestApplyUserAgentHeaderUsingConfig test uses the https://postman-echo.com/ service func TestApplyUserAgentHeaderUsingConfig(t *testing.T) { - type echoBody struct { - Headers map[string]string - } - tmpFile := makeTmpFile(t) - defer os.Remove(tmpFile) - httpClient := http.Client{ - Transport: &roundTripper{UserAgent: "go-downloader / 0.0.0-test"}, - } - config := downloader.Config{ - HttpClient: httpClient, - } - - d, err := downloader.DownloadWithConfig(tmpFile, "https://postman-echo.com/headers", config) + err := downloader.DownloadWithConfig(t.Context(), tmpFile, "https://postman-echo.com/headers", downloader.Config{ + ExtraHeaders: map[string]string{ + "User-Agent": "go-downloader / 0.0.0-test", + }, + }) require.NoError(t, err) - testEchoBody := echoBody{} - body, err := io.ReadAll(d.Resp.Body) + testEchoBody := struct { + Headers map[string]string + }{} + body, err := os.ReadFile(tmpFile) require.NoError(t, err) err = json.Unmarshal(body, &testEchoBody) require.NoError(t, err) @@ -216,23 +244,25 @@ func TestContextCancelation(t *testing.T) { tmpFile := makeTmpFile(t) - ctx, cancel := context.WithCancel(context.Background()) - d, err := downloader.DownloadWithConfigAndContext(ctx, tmpFile, server.URL+"/slow", downloader.Config{}) - require.NoError(t, err) + max := int64(0) + startTime := time.Now() + callback := func(curr, size int64) { + fmt.Println("Downloaded", curr, "bytes of", size, ". Elapsed:", time.Since(startTime)) + max = curr + } + ctx, cancel := context.WithCancel(context.Background()) // Cancel in two seconds - startTime := time.Now() go func() { time.Sleep(2 * time.Second) cancel() }() // Run slow download - max := int64(0) - err = d.RunAndPoll(func(curr int64) { - fmt.Println("Downloaded", curr, "bytes. Elapsed:", time.Since(startTime)) - max = curr - }, 100*time.Millisecond) + err := downloader.DownloadWithConfig(ctx, tmpFile, server.URL+"/slow", downloader.Config{ + PollInterval: 100 * time.Millisecond, + PollFunction: callback, + }) require.EqualError(t, err, "context canceled") require.True(t, max < 210) fmt.Println("Context canceled successfully") @@ -255,9 +285,8 @@ func TestTimeoutOnHEADCall(t *testing.T) { } startTime := time.Now() - d, err := downloader.DownloadWithConfig(tmpFile, server.URL+"/timeout", config) + err := downloader.DownloadWithConfig(t.Context(), tmpFile, server.URL+"/timeout", config) require.Error(t, err) - require.Nil(t, d) require.Contains(t, err.Error(), "context deadline exceeded") elapsed := time.Since(startTime) require.True(t, elapsed < 5*time.Second) @@ -284,9 +313,8 @@ func TestTimeoutOnGETCall(t *testing.T) { } startTime := time.Now() - d, err := downloader.DownloadWithConfig(tmpFile, server.URL+"/slow", config) + err := downloader.DownloadWithConfig(t.Context(), tmpFile, server.URL+"/slow", config) require.Error(t, err) - require.Nil(t, d) require.Contains(t, err.Error(), "i/o timeout") elapsed := time.Since(startTime) require.True(t, elapsed < 5*time.Second) @@ -327,18 +355,15 @@ func TestInactivityTimeout(t *testing.T) { t.Cleanup(server.Close) // Request download with inactivity timeout of 500ms + // (should timeout due to inactivity) tmpFile := makeTmpFile(t) - d, err := downloader.DownloadWithConfigAndContext( + startTime := time.Now() + err := downloader.DownloadWithConfig( t.Context(), tmpFile, server.URL+"/inactive", downloader.Config{ InactivityTimeout: 500 * time.Millisecond, }, ) - require.NoError(t, err) - - // Run the download, should timeout due to inactivity - startTime := time.Now() - err = d.Run() elapsed := time.Since(startTime) // Check that we got a timeout error diff --git a/examples/check_download_size/main.go b/examples/check_download_size/main.go index 2690014..a2c3141 100644 --- a/examples/check_download_size/main.go +++ b/examples/check_download_size/main.go @@ -7,6 +7,7 @@ package main import ( + "context" "fmt" "log" "net/http" @@ -15,15 +16,14 @@ import ( ) func main() { - _, err := downloader.DownloadWithConfig("test.txt", "https://go.bug.st/test.txt", downloader.Config{ + if err := downloader.DownloadWithConfig(context.Background(), "test.txt", "https://go.bug.st/test.txt", downloader.Config{ AcceptFunc: func(head *http.Response) error { if head.ContentLength > 2000 { return fmt.Errorf("insufficient space for download") } return nil }, - }) - if err != nil { + }); err != nil { log.Fatal(err) } diff --git a/examples/download_file/main.go b/examples/download_file/main.go index a33629c..3a93298 100644 --- a/examples/download_file/main.go +++ b/examples/download_file/main.go @@ -7,6 +7,7 @@ package main import ( + "context" "fmt" "log" "os" @@ -23,15 +24,16 @@ func main() { } defer os.RemoveAll(tmp) - d, err := downloader.Download(filepath.Join(tmp, "test.txt"), "https://go.bug.st/test.txt") - if err != nil { - log.Fatal(err) - } - - progressCB := func(current int64) { - fmt.Printf("Downloaded %d / %d bytes (%.2f%%)\n", current, d.Size(), float64(current)*100.0/float64(d.Size())) - } - if err := d.RunAndPoll(progressCB, time.Millisecond); err != nil { + if err := downloader.DownloadWithConfig( + context.Background(), + filepath.Join(tmp, "test.txt"), + "https://go.bug.st/test.txt", + downloader.Config{ + PollInterval: time.Millisecond, + PollFunction: func(current, size int64) { + fmt.Printf("Downloaded %d / %d bytes (%.2f%%)\n", current, size, float64(current)*100.0/float64(size)) + }, + }); err != nil { log.Fatal(err) } }