Skip to content

Commit

Permalink
Make parallel downloads test more strict
Browse files Browse the repository at this point in the history
We can assert now that only one thread downloaded the file, and all
other threads used the cache.

Signed-off-by: Nir Soffer <[email protected]>
  • Loading branch information
nirs committed Nov 14, 2024
1 parent 76d5c86 commit c1b4d72
Showing 1 changed file with 28 additions and 24 deletions.
52 changes: 28 additions & 24 deletions pkg/downloader/downloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"os/exec"
"path/filepath"
"runtime"
"slices"
"strings"
"testing"
"time"
Expand All @@ -31,11 +30,6 @@ type downloadResult struct {
// races quicker. 20 parallel downloads take about 120 milliseconds on M1 Pro.
const parallelDownloads = 20

// When downloading in parallel usually all downloads completed with
// StatusDownload, but some may be delayed and find the data file when they
// start. Can be reproduced locally using 100 parallel downloads.
var parallelStatus = []Status{StatusDownloaded, StatusUsedCache}

func TestDownloadRemote(t *testing.T) {
ts := httptest.NewServer(http.FileServer(http.Dir("testdata")))
t.Cleanup(ts.Close)
Expand Down Expand Up @@ -103,15 +97,10 @@ func TestDownloadRemote(t *testing.T) {
results <- downloadResult{r, err}
}()
}
// We must process all results before cleanup.
for i := 0; i < parallelDownloads; i++ {
result := <-results
if result.err != nil {
t.Errorf("Download failed: %s", result.err)
} else if !slices.Contains(parallelStatus, result.r.Status) {
t.Errorf("Expected download status %s, got %s", parallelStatus, result.r.Status)
}
}
// Only one thread should download, the rest should use the cache.
downloaded, cached := countResults(t, results)
assert.Equal(t, downloaded, 1)
assert.Equal(t, cached, parallelDownloads-1)
})
})
t.Run("caching-only mode", func(t *testing.T) {
Expand Down Expand Up @@ -146,15 +135,10 @@ func TestDownloadRemote(t *testing.T) {
results <- downloadResult{r, err}
}()
}
// We must process all results before cleanup.
for i := 0; i < parallelDownloads; i++ {
result := <-results
if result.err != nil {
t.Errorf("Download failed: %s", result.err)
} else if !slices.Contains(parallelStatus, result.r.Status) {
t.Errorf("Expected download status %s, got %s", parallelStatus, result.r.Status)
}
}
// Only one thread should download, the rest should use the cache.
downloaded, cached := countResults(t, results)
assert.Equal(t, downloaded, 1)
assert.Equal(t, cached, parallelDownloads-1)
})
})
t.Run("cached", func(t *testing.T) {
Expand Down Expand Up @@ -188,6 +172,26 @@ func TestDownloadRemote(t *testing.T) {
})
}

func countResults(t *testing.T, results chan downloadResult) (downloaded, cached int) {
t.Helper()
for i := 0; i < parallelDownloads; i++ {
result := <-results
if result.err != nil {
t.Errorf("Download failed: %s", result.err)
} else {
switch result.r.Status {
case StatusDownloaded:
downloaded++
case StatusUsedCache:
cached++
default:
t.Errorf("Unexpected download status %q", result.r.Status)
}
}
}
return downloaded, cached
}

func TestRedownloadRemote(t *testing.T) {
remoteDir := t.TempDir()
ts := httptest.NewServer(http.FileServer(http.Dir(remoteDir)))
Expand Down

0 comments on commit c1b4d72

Please sign in to comment.