diff --git a/README.md b/README.md index e487fc5..a0fcb65 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ -# go.bug.st/downloader/v2 +# A simple HTTP/S file downloader for golang [![Test Go status](https://github.com/bugst/go-downloader/actions/workflows/go-test.yml/badge.svg)](https://github.com/bugst/go-downloader/actions/workflows/go-test.yml) [![codecov](https://codecov.io/gh/bugst/go-downloader/branch/main/graph/badge.svg)](https://codecov.io/gh/bugst/go-downloader) -A simple HTTP/S file downloader for golang. +See the `examples` folder for some usage example. + +## License + +This library is released under the permissive BSD-3-CLAUSE. diff --git a/downloader.go b/downloader.go index fdbcc7a..0897d19 100644 --- a/downloader.go +++ b/downloader.go @@ -12,7 +12,6 @@ import ( "io" "net/http" "os" - "slices" "sync" "time" ) @@ -29,25 +28,15 @@ type Downloader struct { err error } -// DownloadOptions are optional flags that can be passed to Download function -type DownloadOptions int - -const ( - // NoResume will not try to resume a partial download - NoResume DownloadOptions = iota -) - // Close the download -func (d *Downloader) Close() error { - err1 := d.out.Close() - err2 := d.Resp.Body.Close() - if err1 != nil { - return fmt.Errorf("closing output file: %s", err1) +func (d *Downloader) close() { + if d.out != nil { + d.out.Close() } - if err2 != nil { - return fmt.Errorf("closing input stream: %s", err2) + if d.Resp != nil { + d.Resp.Body.Close() } - return nil + close(d.Done) } // Size return the size of the download (or -1 if the server doesn't provide it) @@ -61,7 +50,7 @@ func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration t := time.NewTicker(interval) defer t.Stop() - go d.AsyncRun() + go d.Run() for { select { case <-t.C: @@ -73,20 +62,11 @@ func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration } } -// AsyncRun starts the downloader copy-loop. This function is supposed to be run -// on his own goroutine, it will close the Done channel when the download is completed -// or an error occurs. -// -// Deprecated: use Run instead. -func (d *Downloader) AsyncRun() { - d.Run() -} - // Run starts the downloader and waits until it completes the download. // This method can be run in a goroutine to perform an asynchronous download; // it will close the Done channel when the download is completed or an error occurs. func (d *Downloader) Run() error { - defer close(d.Done) + defer d.close() d.completedLock.Lock() skip := (d.completed == d.size) @@ -113,7 +93,6 @@ func (d *Downloader) Run() error { break } } - _ = d.Close() return d.Error() } @@ -133,26 +112,26 @@ func (d *Downloader) Completed() int64 { // Download returns an asynchronous downloader that will download the specified url // in the specified file. A download resume is tried if a file shorter than the requested // url is already present. -func Download(file string, reqURL string, options ...DownloadOptions) (*Downloader, error) { - return DownloadWithConfig(file, reqURL, GetDefaultConfig(), options...) +func Download(file string, reqURL string) (*Downloader, error) { + return DownloadWithConfig(file, reqURL, GetDefaultConfig()) } // DownloadWithConfig applies an additional configuration to the http client and // returns an asynchronous downloader that will download the specified url // in the specified file. A download resume is tried if a file shorter than the requested // url is already present. -func DownloadWithConfig(file string, reqURL string, config Config, options ...DownloadOptions) (*Downloader, error) { - return DownloadWithConfigAndContext(context.Background(), file, reqURL, config, options...) +func DownloadWithConfig(file string, reqURL string, config Config) (*Downloader, error) { + return DownloadWithConfigAndContext(context.Background(), file, reqURL, config) } // DownloadWithConfigAndContext applies an additional configuration to the http client and // returns an asynchronous downloader that will download the specified url // in the specified file. // A previous download is resumed if the local file is shorter than the remote file. -// The download is skipped if the local file has the same size of the renote file. +// The download is skipped if the local file has the same size of the remote file. // The download is restarted from scratch if the local file is larger than the remote file. -func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config, options ...DownloadOptions) (*Downloader, error) { - clientCanResume := !slices.Contains(options, NoResume) +func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config) (*Downloader, error) { + clientCanResume := !config.DoNotResumeDownload // Gather information about local file var localSize int64 @@ -165,14 +144,26 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin if err != nil { return nil, fmt.Errorf("setting up HEAD request: %s", err) } + if config.ExtraHeaders != nil { + for k, v := range config.ExtraHeaders { + headReq.Header.Set(k, v) + } + } headResp, err := config.HttpClient.Do(headReq) if err != nil { return nil, fmt.Errorf("performing HEAD request: %s", err) } remoteSize := headResp.ContentLength // -1 if server doesn't send Content-Length serverCanResume := (headResp.Header.Get("Accept-Ranges") == "bytes") && (remoteSize != -1) + var acceptError error + if config.AcceptFunc != nil { + acceptError = config.AcceptFunc(headResp) + } _, _ = io.Copy(io.Discard, headResp.Body) _ = headResp.Body.Close() + if acceptError != nil { + return nil, acceptError + } // If we are allowed to resume a download, check the local file size and decide how to proceed var completed int64 @@ -199,6 +190,11 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin if err != nil { return nil, fmt.Errorf("setting up HTTP request: %s", err) } + if config.ExtraHeaders != nil { + for k, v := range config.ExtraHeaders { + req.Header.Set(k, v) + } + } resumeDownload := clientCanResume && serverCanResume && completed > 0 if resumeDownload { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", completed)) diff --git a/downloader_config.go b/downloader_config.go index d7600d2..11bfda1 100644 --- a/downloader_config.go +++ b/downloader_config.go @@ -13,7 +13,16 @@ import ( // Config contains the configuration for the downloader type Config struct { + // HttpClient to use to perform HTTP requests HttpClient http.Client + // DoNotResumeDownload set to true to disallow resuming downloads. + DoNotResumeDownload bool + // ExtraHeaders to add to the HTTP requests. + ExtraHeaders map[string]string + // AcceptFunc is an optional function that will be called + // when the HTTP HEAD request is done, before starting the download. + // If the function returns an error, the download is aborted. + AcceptFunc func(head *http.Response) error } var defaultConfig Config = Config{} diff --git a/downloader_test.go b/downloader_test.go index 5537f7c..dc690b6 100644 --- a/downloader_test.go +++ b/downloader_test.go @@ -18,7 +18,7 @@ import ( "time" "github.com/stretchr/testify/require" - "go.bug.st/downloader/v2" + "go.bug.st/downloader/v3" ) func makeTmpFile(t *testing.T) string { @@ -102,7 +102,9 @@ func TestNoResume(t *testing.T) { err = os.WriteFile(tmpFile, part, 0644) require.NoError(t, err) - d, err := downloader.Download(tmpFile, "https://go.bug.st/test.txt", downloader.NoResume) + d, err := downloader.DownloadWithConfig(tmpFile, "https://go.bug.st/test.txt", downloader.Config{ + DoNotResumeDownload: true, + }) require.Equal(t, int64(0), d.Completed()) require.Equal(t, int64(8052), d.Size()) require.NoError(t, err) diff --git a/examples/check_download_size/main.go b/examples/check_download_size/main.go new file mode 100644 index 0000000..29d61c7 --- /dev/null +++ b/examples/check_download_size/main.go @@ -0,0 +1,31 @@ +// +// Copyright 2025 Cristian Maglie. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// + +package main + +import ( + "fmt" + "log" + "net/http" + + "go.bug.st/downloader/v3" +) + +func main() { + _, err := downloader.DownloadWithConfig("test.txt", "https://go.bug.st/test.txt", downloader.Config{ + AcceptFunc: func(head *http.Response) error { + if head.ContentLength > 2000 { + return fmt.Errorf("insufficient space for download") + } + return nil + }, + }) + if err != nil { + log.Fatal(err) + } + + panic("Unreachable, should exit on AcceptFunc") +} diff --git a/examples/download_file/main.go b/examples/download_file/main.go new file mode 100644 index 0000000..491f415 --- /dev/null +++ b/examples/download_file/main.go @@ -0,0 +1,37 @@ +// +// Copyright 2025 Cristian Maglie. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// + +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + "time" + + "go.bug.st/downloader/v3" +) + +func main() { + tmp, err := os.MkdirTemp("", "") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(tmp) + + d, err := downloader.Download(filepath.Join(tmp, "test.txt"), "https://go.bug.st/test.txt") + if err != nil { + log.Fatal(err) + } + + progressCB := func(current int64) { + fmt.Printf("Downloaded %d / %d bytes (%.2f%%)\n", current, d.Size(), float64(current)*100.0/float64(d.Size())) + } + if err := d.RunAndPoll(progressCB, time.Millisecond); err != nil { + log.Fatal(err) + } +} diff --git a/go.mod b/go.mod index e2204f5..aaf0d29 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.bug.st/downloader/v2 +module go.bug.st/downloader/v3 go 1.24