Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
# go.bug.st/downloader/v2
# A simple HTTP/S file downloader for golang

[![Test Go status](https://github.com/bugst/go-downloader/actions/workflows/go-test.yml/badge.svg)](https://github.com/bugst/go-downloader/actions/workflows/go-test.yml)

[![codecov](https://codecov.io/gh/bugst/go-downloader/branch/main/graph/badge.svg)](https://codecov.io/gh/bugst/go-downloader)

A simple HTTP/S file downloader for golang.
See the `examples` folder for some usage example.

## License

This library is released under the permissive BSD-3-CLAUSE.
68 changes: 32 additions & 36 deletions downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"io"
"net/http"
"os"
"slices"
"sync"
"time"
)
Expand All @@ -29,25 +28,15 @@ type Downloader struct {
err error
}

// DownloadOptions are optional flags that can be passed to Download function
type DownloadOptions int

const (
// NoResume will not try to resume a partial download
NoResume DownloadOptions = iota
)

// Close the download
func (d *Downloader) Close() error {
err1 := d.out.Close()
err2 := d.Resp.Body.Close()
if err1 != nil {
return fmt.Errorf("closing output file: %s", err1)
func (d *Downloader) close() {
if d.out != nil {
d.out.Close()
}
if err2 != nil {
return fmt.Errorf("closing input stream: %s", err2)
if d.Resp != nil {
d.Resp.Body.Close()
}
return nil
close(d.Done)
}

// Size return the size of the download (or -1 if the server doesn't provide it)
Expand All @@ -61,7 +50,7 @@ func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration
t := time.NewTicker(interval)
defer t.Stop()

go d.AsyncRun()
go d.Run()
for {
select {
case <-t.C:
Expand All @@ -73,20 +62,11 @@ func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration
}
}

// AsyncRun starts the downloader copy-loop. This function is supposed to be run
// on his own goroutine, it will close the Done channel when the download is completed
// or an error occurs.
//
// Deprecated: use Run instead.
func (d *Downloader) AsyncRun() {
d.Run()
}

// Run starts the downloader and waits until it completes the download.
// This method can be run in a goroutine to perform an asynchronous download;
// it will close the Done channel when the download is completed or an error occurs.
func (d *Downloader) Run() error {
defer close(d.Done)
defer d.close()

d.completedLock.Lock()
skip := (d.completed == d.size)
Expand All @@ -113,7 +93,6 @@ func (d *Downloader) Run() error {
break
}
}
_ = d.Close()
return d.Error()
}

Expand All @@ -133,26 +112,26 @@ func (d *Downloader) Completed() int64 {
// Download returns an asynchronous downloader that will download the specified url
// in the specified file. A download resume is tried if a file shorter than the requested
// url is already present.
func Download(file string, reqURL string, options ...DownloadOptions) (*Downloader, error) {
return DownloadWithConfig(file, reqURL, GetDefaultConfig(), options...)
func Download(file string, reqURL string) (*Downloader, error) {
return DownloadWithConfig(file, reqURL, GetDefaultConfig())
}

// DownloadWithConfig applies an additional configuration to the http client and
// returns an asynchronous downloader that will download the specified url
// in the specified file. A download resume is tried if a file shorter than the requested
// url is already present.
func DownloadWithConfig(file string, reqURL string, config Config, options ...DownloadOptions) (*Downloader, error) {
return DownloadWithConfigAndContext(context.Background(), file, reqURL, config, options...)
func DownloadWithConfig(file string, reqURL string, config Config) (*Downloader, error) {
return DownloadWithConfigAndContext(context.Background(), file, reqURL, config)
}

// DownloadWithConfigAndContext applies an additional configuration to the http client and
// returns an asynchronous downloader that will download the specified url
// in the specified file.
// A previous download is resumed if the local file is shorter than the remote file.
// The download is skipped if the local file has the same size of the renote file.
// The download is skipped if the local file has the same size of the remote file.
// The download is restarted from scratch if the local file is larger than the remote file.
func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config, options ...DownloadOptions) (*Downloader, error) {
clientCanResume := !slices.Contains(options, NoResume)
func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config) (*Downloader, error) {
clientCanResume := !config.DoNotResumeDownload

// Gather information about local file
var localSize int64
Expand All @@ -165,14 +144,26 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
if err != nil {
return nil, fmt.Errorf("setting up HEAD request: %s", err)
}
if config.ExtraHeaders != nil {
for k, v := range config.ExtraHeaders {
headReq.Header.Set(k, v)
}
}
headResp, err := config.HttpClient.Do(headReq)
if err != nil {
return nil, fmt.Errorf("performing HEAD request: %s", err)
}
remoteSize := headResp.ContentLength // -1 if server doesn't send Content-Length
serverCanResume := (headResp.Header.Get("Accept-Ranges") == "bytes") && (remoteSize != -1)
var acceptError error
if config.AcceptFunc != nil {
acceptError = config.AcceptFunc(headResp)
}
_, _ = io.Copy(io.Discard, headResp.Body)
_ = headResp.Body.Close()
if acceptError != nil {
return nil, acceptError
}

// If we are allowed to resume a download, check the local file size and decide how to proceed
var completed int64
Expand All @@ -199,6 +190,11 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
if err != nil {
return nil, fmt.Errorf("setting up HTTP request: %s", err)
}
if config.ExtraHeaders != nil {
for k, v := range config.ExtraHeaders {
req.Header.Set(k, v)
}
}
resumeDownload := clientCanResume && serverCanResume && completed > 0
if resumeDownload {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", completed))
Expand Down
9 changes: 9 additions & 0 deletions downloader_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,16 @@ import (

// Config contains the configuration for the downloader
type Config struct {
// HttpClient to use to perform HTTP requests
HttpClient http.Client
// DoNotResumeDownload set to true to disallow resuming downloads.
DoNotResumeDownload bool
// ExtraHeaders to add to the HTTP requests.
ExtraHeaders map[string]string
// AcceptFunc is an optional function that will be called
// when the HTTP HEAD request is done, before starting the download.
// If the function returns an error, the download is aborted.
AcceptFunc func(head *http.Response) error
}

var defaultConfig Config = Config{}
Expand Down
6 changes: 4 additions & 2 deletions downloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"time"

"github.com/stretchr/testify/require"
"go.bug.st/downloader/v2"
"go.bug.st/downloader/v3"
)

func makeTmpFile(t *testing.T) string {
Expand Down Expand Up @@ -102,7 +102,9 @@ func TestNoResume(t *testing.T) {
err = os.WriteFile(tmpFile, part, 0644)
require.NoError(t, err)

d, err := downloader.Download(tmpFile, "https://go.bug.st/test.txt", downloader.NoResume)
d, err := downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{
DoNotResumeDownload: true,
})
require.Equal(t, int64(0), d.Completed())
require.Equal(t, int64(8052), d.Size())
require.NoError(t, err)
Expand Down
31 changes: 31 additions & 0 deletions examples/check_download_size/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
//
// Copyright 2025 Cristian Maglie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package main

import (
"fmt"
"log"
"net/http"

"go.bug.st/downloader/v3"
)

func main() {
_, err := downloader.DownloadWithConfig("test.txt", "https://go.bug.st/test.txt", downloader.Config{
AcceptFunc: func(head *http.Response) error {
if head.ContentLength > 2000 {
return fmt.Errorf("insufficient space for download")
}
return nil
},
})
if err != nil {
log.Fatal(err)
}

panic("Unreachable, should exit on AcceptFunc")
}
37 changes: 37 additions & 0 deletions examples/download_file/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
//
// Copyright 2025 Cristian Maglie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package main

import (
"fmt"
"log"
"os"
"path/filepath"
"time"

"go.bug.st/downloader/v3"
)

func main() {
tmp, err := os.MkdirTemp("", "")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(tmp)

d, err := downloader.Download(filepath.Join(tmp, "test.txt"), "https://go.bug.st/test.txt")
if err != nil {
log.Fatal(err)
}

progressCB := func(current int64) {
fmt.Printf("Downloaded %d / %d bytes (%.2f%%)\n", current, d.Size(), float64(current)*100.0/float64(d.Size()))
}
if err := d.RunAndPoll(progressCB, time.Millisecond); err != nil {
log.Fatal(err)
}
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
module go.bug.st/downloader/v2
module go.bug.st/downloader/v3

go 1.24

Expand Down