Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 87 additions & 158 deletions downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,130 +7,17 @@
package downloader

import (
"cmp"
"context"
"fmt"
"io"
"net/http"
"os"
"sync"
"sync/atomic"
"time"
)

// Downloader is an asynchronous downloader
type Downloader struct {
URL string
Resp *http.Response
out *os.File
completed int64
completedLock sync.Mutex
size int64
err error
wdog watchdog
}

// Size return the size of the download (or -1 if the server doesn't provide it)
func (d *Downloader) Size() int64 {
return d.size
}

// RunAndPoll starts the downloader copy-loop and calls the poll function every
// interval time to update progress.
func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration) error {
t := time.NewTicker(interval)
defer t.Stop()

res := make(chan error, 1)
go func() {
res <- d.Run()
}()
for {
select {
case <-t.C:
poll(d.Completed())
case err := <-res:
poll(d.Completed())
return err
}
}
}

// Run starts the downloader and waits until it completes the download.
// This method can be run in a goroutine to perform an asynchronous download;
// it will close the Done channel when the download is completed or an error occurs.
func (d *Downloader) Run() error {
d.completedLock.Lock()
skip := (d.completed == d.size)
d.completedLock.Unlock()
if skip {
return d.Error()
}

defer func() {
d.out.Close()
d.Resp.Body.Close()
d.wdog.Cancel()
}()

in := d.Resp.Body
buff := [4096]byte{}
for {
n, readErr := in.Read(buff[:])
if n > 0 {
if _, writeErr := d.out.Write(buff[:n]); writeErr != nil {
// Error writing to file
d.err = writeErr
break
}

// Update completed bytes count
d.completedLock.Lock()
d.completed += int64(n)
d.completedLock.Unlock()

// Extend inactivity timeout deadline
d.wdog.Kick()
}
if readErr == io.EOF {
// Download completed successfully!
break
}
if readErr != nil {
// Network or other error
d.err = readErr
break
}
}
return d.Error()
}

// Error returns the error during download or nil if no errors happened
func (d *Downloader) Error() error {
return d.err
}

// Completed returns the bytes read so far
func (d *Downloader) Completed() int64 {
d.completedLock.Lock()
res := d.completed
d.completedLock.Unlock()
return res
}

// Download returns an asynchronous downloader that will download the specified url
// in the specified file. A download resume is tried if a file shorter than the requested
// url is already present.
func Download(file string, reqURL string) (*Downloader, error) {
return DownloadWithConfig(file, reqURL, GetDefaultConfig())
}

// DownloadWithConfig applies an additional configuration to the http client and
// returns an asynchronous downloader that will download the specified url
// in the specified file. A download resume is tried if a file shorter than the requested
// url is already present.
func DownloadWithConfig(file string, reqURL string, config Config) (*Downloader, error) {
return DownloadWithConfigAndContext(context.Background(), file, reqURL, config)
}

func doHeadRequest(ctx context.Context, reqURL string, config Config) (*http.Response, error) {
headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil)
if err != nil {
Expand All @@ -156,13 +43,20 @@ func doHeadRequest(ctx context.Context, reqURL string, config Config) (*http.Res
return headResp, nil
}

// DownloadWithConfigAndContext applies an additional configuration to the http client and
// returns an asynchronous downloader that will download the specified url
// in the specified file.
// A previous download is resumed if the local file is shorter than the remote file.
// Download downloads the specified url in the specified file.
// A download resume is tried if a file shorter than the requested url is already present.
func Download(ctx context.Context, file string, reqURL string) error {
return DownloadWithConfig(ctx, file, reqURL, GetDefaultConfig())
}

// DownloadWithConfig applies an additional configuration to the http client and
// downloads download the specified url in the specified file.
// A previous download is resumed if the local file is shorter than the remote file,
// unless otherwise specified.
// The download is skipped if the local file has the same size of the remote file.
// The download is restarted from scratch if the local file is larger than the remote file.
func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config) (*Downloader, error) {
// The download is restarted from scratch if the local file is larger than the remote
// file or if the server doesn't support resuming.
func DownloadWithConfig(ctx context.Context, file string, reqURL string, config Config) error {
clientCanResume := !config.DoNotResumeDownload

// Gather information about local file
Expand All @@ -174,7 +68,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
// Perform a HEAD call to gather information about the server capabilities and remote file
headResp, err := doHeadRequest(ctx, reqURL, config)
if err != nil {
return nil, err
return err
}
remoteSize := headResp.ContentLength // -1 if server doesn't send Content-Length
serverCanResume := (headResp.Header.Get("Accept-Ranges") == "bytes") && (remoteSize != -1)
Expand All @@ -185,60 +79,51 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
acceptError = config.AcceptFunc(headResp)
}
if acceptError != nil {
return nil, acceptError
return acceptError
}

// If we are allowed to resume a download, check the local file size and decide how to proceed
var completed int64
if clientCanResume {
if localSize == remoteSize {
// Size matches: assume the file is already downloaded
return &Downloader{
URL: reqURL,
Resp: headResp,
completed: remoteSize,
size: remoteSize,
}, nil
if config.PollFunction != nil {
config.PollFunction(remoteSize, remoteSize)
}
return nil
}
if localSize < remoteSize {
// Local file is smaller than remote file: resume download
// Remote size is unknown: resume download anyway
completed = localSize
if localSize > remoteSize {
// Local file is larger than remote file: restart download
clientCanResume = false
}
}

// Perform the actual GET request
// Setup inactivity timeout for the GET request
ctx, wdog := newWatchdog(ctx, config.InactivityTimeout)
defer wdog.Cancel()
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
if err != nil {
wdog.Cancel()
return nil, fmt.Errorf("setting up HTTP request: %s", err)
return fmt.Errorf("setting up HTTP request: %s", err)
}
if config.ExtraHeaders != nil {
for k, v := range config.ExtraHeaders {
req.Header.Set(k, v)
}
}
resumeDownload := clientCanResume && serverCanResume && completed > 0
resumeDownload := clientCanResume && serverCanResume && localSize > 0
if resumeDownload {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", completed))
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", localSize))
}
resp, err := config.HttpClient.Do(req)
if err != nil {
wdog.Cancel()
return nil, err
return err
}
defer resp.Body.Close()

// Check server response
if !config.DoNotErrorOnNon2xxStatusCode {
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
_ = resp.Body.Close()
wdog.Cancel()
return &Downloader{
URL: reqURL,
Resp: resp, // Return response for further inspection
}, fmt.Errorf("server returned %s", resp.Status)
return fmt.Errorf("server returned %s", resp.Status)
}
}

Expand All @@ -249,19 +134,63 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
} else {
flags |= os.O_CREATE | os.O_TRUNC
}
f, err := os.OpenFile(file, flags, 0644)
out, err := os.OpenFile(file, flags, 0644)
if err != nil {
_ = resp.Body.Close()
wdog.Cancel()
return nil, fmt.Errorf("opening %s for writing: %s", file, err)
return fmt.Errorf("opening %s for writing: %s", file, err)
}
defer out.Close()

// Start polling goroutine if requested
var completed atomic.Int64
if resumeDownload {
completed.Store(localSize)
}
if config.PollFunction != nil {
var updateLock sync.Mutex
update := func() {
updateLock.Lock()
defer updateLock.Unlock()
config.PollFunction(completed.Load(), remoteSize)
}

// send initial update
update()

return &Downloader{
URL: reqURL,
Resp: resp,
out: f,
completed: completed,
size: remoteSize,
wdog: wdog,
}, nil
interval := cmp.Or(config.PollInterval.Abs(), 250*time.Millisecond)
var t *time.Timer
t = time.AfterFunc(interval, func() {
// send intermediate updates
update()
t.Reset(interval)
})

defer func() {
t.Stop()
// send final update
update()
}()
}

in := resp.Body
buff := [4096]byte{}
for {
n, readErr := in.Read(buff[:])
if n > 0 {
if _, writeErr := out.Write(buff[:n]); writeErr != nil {
// Error writing to file
return writeErr
}

completed.Add(int64(n))

// Extend inactivity timeout deadline
wdog.Kick()
}
if readErr == io.EOF {
return nil
}
if readErr != nil {
return readErr
}
}
}
5 changes: 5 additions & 0 deletions downloader_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ type Config struct {
// InactivityTimeout is the duration after which, if no data is received,
// the download is aborted. If set to 0, no timeout is applied.
InactivityTimeout time.Duration
// PollFunction is an optional function that will be called periodically
// during the download to report progress.
PollFunction func(current, size int64)
// PollInterval is the interval at which the PollFunction is called (default is 250ms if not specified).
PollInterval time.Duration
}

var defaultConfig Config = Config{}
Expand Down
Loading