Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
testdata/* text eol=lf
1 change: 1 addition & 0 deletions .github/workflows/go-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ jobs:
go-version-file: go.mod

- name: Run tests
shell: bash
run: go test -race -coverprofile=coverage.txt -covermode=atomic
26 changes: 18 additions & 8 deletions downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"io"
"net/http"
"os"
"slices"
"sync"
"time"
)
Expand Down Expand Up @@ -137,20 +138,29 @@ func DownloadWithConfig(file string, reqURL string, config Config, options ...Do
// in the specified file. A download resume is tried if a file shorter than the requested
// url is already present. The download can be cancelled using the provided context.
func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config, options ...DownloadOptions) (*Downloader, error) {
noResume := false
for _, opt := range options {
if opt == NoResume {
noResume = true
}
noResume := slices.Contains(options, NoResume)

// Perform a HEAD call to gather information about the server capabilities
serverAcceptsRanges := false
var size int64
if headReq, err := http.NewRequestWithContext(ctx, "HEAD", reqURL, nil); err != nil {
return nil, fmt.Errorf("setting up HEAD request: %s", err)
} else if headResp, err := config.HttpClient.Do(headReq); err != nil {
return nil, fmt.Errorf("performing HEAD request: %s", err)
} else {
serverAcceptsRanges = (headResp.Header.Get("Accept-Ranges") == "bytes")
size = headResp.ContentLength
_ = headResp.Body.Close()
}
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)

// Setup the actual GET request
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
if err != nil {
return nil, fmt.Errorf("setting up HTTP request: %s", err)
}

var completed int64
if !noResume {
if !noResume && serverAcceptsRanges {
Comment thread
cmaglie marked this conversation as resolved.
Resolved
if info, err := os.Stat(file); err == nil {
completed = info.Size()
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", completed))
Expand Down Expand Up @@ -182,7 +192,7 @@ func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL strin
Resp: resp,
out: f,
completed: completed,
size: resp.ContentLength + completed,
size: size,
}
return d, nil
}
48 changes: 32 additions & 16 deletions downloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"io"
"net/http"
"os"
"path/filepath"
"testing"
"time"

Expand All @@ -25,12 +26,14 @@ func makeTmpFile(t *testing.T) string {
require.NoError(t, tmp.Close())
tmpFile := tmp.Name()
require.NoError(t, os.Remove(tmpFile))
t.Cleanup(func() {
os.Remove(tmpFile)
})
return tmpFile
}

func TestDownload(t *testing.T) {
tmpFile := makeTmpFile(t)
defer os.Remove(tmpFile)

d, err := Download(tmpFile, "https://go.bug.st/test.txt")
require.NoError(t, err)
Expand All @@ -49,7 +52,6 @@ func TestDownload(t *testing.T) {

func TestResume(t *testing.T) {
tmpFile := makeTmpFile(t)
defer os.Remove(tmpFile)

part, err := os.ReadFile("testdata/test.txt.part")
require.NoError(t, err)
Expand All @@ -71,9 +73,28 @@ func TestResume(t *testing.T) {
require.Equal(t, file1, file2)
}

func TestResumeOnAlreadyCompletedFile(t *testing.T) {
tmpFile := makeTmpFile(t)

full, err := os.ReadFile("testdata/test.txt")
require.NoError(t, err)
require.NoError(t, os.WriteFile(tmpFile, full, 0644))

d, err := Download(tmpFile, "https://go.bug.st/test.txt")
require.NoError(t, err)
require.Equal(t, int64(8052), d.Completed())
require.Equal(t, int64(8052), d.Size())
require.NoError(t, d.Run())
require.Equal(t, int64(8052), d.Completed())

// Check file content is unchanged
file2, err := os.ReadFile(tmpFile)
require.NoError(t, err)
require.Equal(t, full, file2)
}

func TestNoResume(t *testing.T) {
tmpFile := makeTmpFile(t)
defer os.Remove(tmpFile)

part, err := os.ReadFile("testdata/test.txt.part")
require.NoError(t, err)
Expand All @@ -97,7 +118,6 @@ func TestNoResume(t *testing.T) {

func TestInvalidRequest(t *testing.T) {
tmpFile := makeTmpFile(t)
defer os.Remove(tmpFile)

d, err := Download(tmpFile, "asd://go.bug.st/test.txt")
require.Error(t, err)
Expand All @@ -112,7 +132,6 @@ func TestInvalidRequest(t *testing.T) {

func TestRunAndPool(t *testing.T) {
tmpFile := makeTmpFile(t)
defer os.Remove(tmpFile)

d, err := Download(tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2")
require.NoError(t, err)
Expand All @@ -130,11 +149,8 @@ func TestRunAndPool(t *testing.T) {
}

func TestErrorOnFileOpening(t *testing.T) {
tmpFile := makeTmpFile(t)
defer os.Remove(tmpFile)

require.NoError(t, os.WriteFile(tmpFile, []byte{}, 0000))
d, err := Download(tmpFile, "http://go.bug.st/test.txt")
unaccessibleFile := filepath.Join(os.TempDir(), "nonexistentdir", "test.txt")
d, err := Download(unaccessibleFile, "http://go.bug.st/test.txt")
require.Error(t, err)
require.Nil(t, d)
}
Expand Down Expand Up @@ -178,7 +194,7 @@ func TestApplyUserAgentHeaderUsingConfig(t *testing.T) {

func TestContextCancelation(t *testing.T) {
slowHandler := func(w http.ResponseWriter, r *http.Request) {
for i := 0; i < 50; i++ {
for i := range 50 {
fmt.Fprintf(w, "Hello %d\n", i)
w.(http.Flusher).Flush()
time.Sleep(100 * time.Millisecond)
Expand All @@ -189,20 +205,20 @@ func TestContextCancelation(t *testing.T) {
mux.HandleFunc("/slow", slowHandler)
server := &http.Server{Addr: ":8080", Handler: mux}
go func() {
server.ListenAndServe()
fmt.Println("Server stopped")
err := server.ListenAndServe()
fmt.Println("Server stopped with err:", err)
}()
// Wait for server start
time.Sleep(time.Second)

tmpFile := makeTmpFile(t)
defer os.Remove(tmpFile)

ctx, cancel := context.WithCancel(context.Background())
d, err := DownloadWithConfigAndContext(ctx, tmpFile, "http://127.0.0.1:8080/slow", Config{})
require.NoError(t, err)

// Cancel in two seconds
startTime := time.Now()
go func() {
time.Sleep(2 * time.Second)
cancel()
Expand All @@ -211,11 +227,11 @@ func TestContextCancelation(t *testing.T) {
// Run slow download
max := int64(0)
err = d.RunAndPoll(func(curr int64) {
fmt.Println(curr)
fmt.Println("Downloaded", curr, "bytes. Elapsed:", time.Since(startTime))
max = curr
}, 100*time.Millisecond)
require.EqualError(t, err, "context canceled")
require.True(t, max < 400)

require.NoError(t, server.Shutdown(ctx))
require.NoError(t, server.Shutdown(context.Background()))
}