Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 60 additions & 31 deletions downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func (d *Downloader) Close() error {
return nil
}

// Size return the size of the download
// Size return the size of the download (or -1 if the server doesn't provide it)
func (d *Downloader) Size() int64 {
return d.size
}
Expand Down Expand Up @@ -88,6 +88,13 @@ func (d *Downloader) AsyncRun() {
func (d *Downloader) Run() error {
defer close(d.Done)

d.completedLock.Lock()
skip := (d.completed == d.size)
d.completedLock.Unlock()
if skip {
return d.Error()
}
Comment on lines +91 to +96

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this correct? Isn't size and completed both zero the first time?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

size is the size of the remote file, it will be > 0.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering what happens if the server does NOT report the content-size... 🤔

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

field ContentLength int64

ContentLength records the length of the associated content. The value -1 indicates that the length is unknown. Unless Request.Method is "HEAD", values >= 0 indicate that the given number of bytes may be read from Body.

So, theoretically, we could also have -1 in the size field, mmmmmhhhh...

@cmaglie cmaglie Dec 19, 2025 •

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've pushed a fix to handle also the case with the server not providing Content-Lenght.
There are actually 4 cases:

Range= header accetpted Content-Len provided action
N N Restart from beginning (cannot use Range=...)
N Y Restart from beginning (cannot use Range=...)
Y N Always resume download
Restart from beginning
Y Y Resume download only if local file is smaller

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that the Range= no Content-Len= is not correct. In that case you cannot resume the download because if you do a range query with a size that is the fully downloaded file, you will get a 416 Range Not Satisfiable. I think that you can safely resume only if you have both headers.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There you go


in := d.Resp.Body
buff := [4096]byte{}
for {
Expand Down Expand Up @@ -140,64 +147,86 @@ func DownloadWithConfig(file string, reqURL string, config Config, options ...Do

// DownloadWithConfigAndContext applies an additional configuration to the http client and
// returns an asynchronous downloader that will download the specified url
// in the specified file. A download resume is tried if a file shorter than the requested
// url is already present. The download can be cancelled using the provided context.
// in the specified file.
// A previous download is resumed if the local file is shorter than the remote file.
// The download is skipped if the local file has the same size of the renote file.
// The download is restarted from scratch if the local file is larger than the remote file.
func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config, options ...DownloadOptions) (*Downloader, error) {
noResume := slices.Contains(options, NoResume)
clientCanResume := !slices.Contains(options, NoResume)

// Perform a HEAD call to gather information about the server capabilities
serverAcceptsRanges := false
var size int64
if headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil); err != nil {
return nil, fmt.Errorf("setting up HEAD request: %s", err)
} else if headResp, err := config.HttpClient.Do(headReq); err != nil {
return nil, fmt.Errorf("performing HEAD request: %s", err)
} else {
serverAcceptsRanges = (headResp.Header.Get("Accept-Ranges") == "bytes")
size = headResp.ContentLength
_ = headResp.Body.Close()
// Gather information about local file
var localSize int64
if info, err := os.Stat(file); err == nil {
localSize = info.Size()
}

// Setup the actual GET request
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
// Perform a HEAD call to gather information about the server capabilities and remote file
headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil)
if err != nil {
return nil, fmt.Errorf("setting up HTTP request: %s", err)
return nil, fmt.Errorf("setting up HEAD request: %s", err)
}
headResp, err := config.HttpClient.Do(headReq)
if err != nil {
return nil, fmt.Errorf("performing HEAD request: %s", err)
}
remoteSize := headResp.ContentLength // -1 if server doesn't send Content-Length
serverCanResume := (headResp.Header.Get("Accept-Ranges") == "bytes") && (remoteSize != -1)
_, _ = io.Copy(io.Discard, headResp.Body)
_ = headResp.Body.Close()

// If we are allowed to resume a download, check the local file size and decide how to proceed
var completed int64
if !noResume && serverAcceptsRanges {
if info, err := os.Stat(file); err == nil {
completed = info.Size()
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", completed))
if clientCanResume {
if localSize == remoteSize {
// Size matches: assume the file is already downloaded
return &Downloader{
URL: reqURL,
Done: make(chan struct{}),
Resp: headResp,
completed: remoteSize,
size: remoteSize,
}, nil
}
if localSize < remoteSize {
// Local file is smaller than remote file: resume download
// Remote size is unknown: resume download anyway
completed = localSize
}
}

// Perform the actual GET request
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
if err != nil {
return nil, fmt.Errorf("setting up HTTP request: %s", err)
}
resumeDownload := clientCanResume && serverCanResume && completed > 0
if resumeDownload {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", completed))
}
resp, err := config.HttpClient.Do(req)
if err != nil {
return nil, err
}

// TODO: if file size == header size return nil, nil

// Open output file
flags := os.O_WRONLY
if completed == 0 {
flags |= os.O_CREATE | os.O_TRUNC
} else {
if resumeDownload {
flags |= os.O_APPEND
} else {
flags |= os.O_CREATE | os.O_TRUNC
}
f, err := os.OpenFile(file, flags, 0644)
if err != nil {
_ = resp.Body.Close()
return nil, fmt.Errorf("opening %s for writing: %s", file, err)
}

d := &Downloader{
return &Downloader{
URL: reqURL,
Done: make(chan struct{}),
Resp: resp,
out: f,
completed: completed,
size: size,
}
return d, nil
size: remoteSize,
}, nil
}