Skip to content

Commit

Permalink
fix: GZip the binaries before sending
Browse files Browse the repository at this point in the history
  • Loading branch information
@rangoo94
rangoo94 committed Oct 25, 2024
1 parent c8a82a5 commit a535434
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 42 deletions.
42 changes: 6 additions & 36 deletions cmd/tcl/kubectl-testkube/devbox/command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,12 +231,7 @@ func NewDevBoxCommand() *cobra.Command {
fmt.Println("Uploading binaries...")
g.Go(func() error {
its := time.Now()
file, err := os.Open(agentBin.Path())
if err != nil {
return err
}
defer file.Close()
err = objectStorage.Upload(ctx, "bin/testkube-api-server", file, agentBin.Hash())
err = objectStorage.Upload(ctx, "bin/testkube-api-server", agentBin.Path(), agentBin.Hash())
if err != nil {
fmt.Printf("Agent: upload finished in %s. Error: %s\n", time.Since(its).Truncate(time.Millisecond), err)
} else {
Expand All @@ -246,12 +241,7 @@ func NewDevBoxCommand() *cobra.Command {
})
g.Go(func() error {
its := time.Now()
file, err := os.Open(toolkitBin.Path())
if err != nil {
return err
}
defer file.Close()
err = objectStorage.Upload(ctx, "bin/toolkit", file, toolkitBin.Hash())
err = objectStorage.Upload(ctx, "bin/toolkit", toolkitBin.Path(), toolkitBin.Hash())
if err != nil {
fmt.Printf("Toolkit: upload finished in %s. Error: %s\n", time.Since(its).Truncate(time.Millisecond), err)
} else {
Expand All @@ -261,12 +251,7 @@ func NewDevBoxCommand() *cobra.Command {
})
g.Go(func() error {
its := time.Now()
file, err := os.Open(initProcessBin.Path())
if err != nil {
return err
}
defer file.Close()
err = objectStorage.Upload(ctx, "bin/init", file, initProcessBin.Hash())
err = objectStorage.Upload(ctx, "bin/init", initProcessBin.Path(), initProcessBin.Hash())
if err != nil {
fmt.Printf("Init Process: upload finished in %s. Error: %s\n", time.Since(its).Truncate(time.Millisecond), err)
} else {
Expand Down Expand Up @@ -410,12 +395,7 @@ func NewDevBoxCommand() *cobra.Command {
fmt.Printf("Agent: build finished in %s.\n", time.Since(its).Truncate(time.Millisecond))

its = time.Now()
file, err := os.Open(agentBin.Path())
if err != nil {
return err
}
defer file.Close()
err = objectStorage.Upload(ctx, "bin/testkube-api-server", file, agentBin.Hash())
err = objectStorage.Upload(ctx, "bin/testkube-api-server", agentBin.Path(), agentBin.Hash())
if err != nil {
fmt.Printf("Agent: upload finished in %s. Error: %s\n", time.Since(its).Truncate(time.Millisecond), err)
return err
Expand Down Expand Up @@ -453,12 +433,7 @@ func NewDevBoxCommand() *cobra.Command {
fmt.Printf("Toolkit: build finished in %s.\n", time.Since(its).Truncate(time.Millisecond))

its = time.Now()
file, err := os.Open(toolkitBin.Path())
if err != nil {
return err
}
defer file.Close()
err = objectStorage.Upload(ctx, "bin/toolkit", file, toolkitBin.Hash())
err = objectStorage.Upload(ctx, "bin/toolkit", toolkitBin.Path(), toolkitBin.Hash())
if err != nil {
fmt.Printf("Toolkit: upload finished in %s. Error: %s\n", time.Since(its).Truncate(time.Millisecond), err)
return err
Expand All @@ -476,12 +451,7 @@ func NewDevBoxCommand() *cobra.Command {
fmt.Printf("Init Process: build finished in %s.\n", time.Since(its).Truncate(time.Millisecond))

its = time.Now()
file, err := os.Open(initProcessBin.Path())
if err != nil {
return err
}
defer file.Close()
err = objectStorage.Upload(ctx, "bin/init", file, initProcessBin.Hash())
err = objectStorage.Upload(ctx, "bin/init", initProcessBin.Path(), initProcessBin.Hash())
if err != nil {
fmt.Printf("Init Process: upload finished in %s. Error: %s\n", time.Since(its).Truncate(time.Millisecond), err)
return err
Expand Down
39 changes: 33 additions & 6 deletions cmd/tcl/kubectl-testkube/devbox/devutils/objectstorage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"

Expand All @@ -23,6 +25,7 @@ import (

minio2 "github.com/minio/minio-go/v7"

"github.com/kubeshop/testkube/cmd/testworkflow-toolkit/artifacts"
"github.com/kubeshop/testkube/internal/common"
"github.com/kubeshop/testkube/pkg/log"
"github.com/kubeshop/testkube/pkg/storage/minio"
Expand Down Expand Up @@ -154,29 +157,53 @@ func (r *ObjectStorage) WaitForReady(ctx context.Context) error {
}

// TODO: Compress on-fly
func (r *ObjectStorage) Upload(ctx context.Context, path string, reader io.Reader, hash string) error {
func (r *ObjectStorage) Upload(ctx context.Context, path string, fsPath string, hash string) error {
c, err := r.Client()
if err != nil {
return err
}
if hash != "" && r.Is(path, hash) {
return nil
}
putUrl, err := c.PresignedPutObject(ctx, "devbox", path, 15*time.Minute)
//putUrl, err := c.PresignedPutObject(ctx, "devbox", path, 15*time.Minute)
putUrl, err := c.PresignHeader(ctx, "PUT", "devbox", path, 15*time.Minute, nil, http.Header{
"X-Amz-Meta-Snowball-Auto-Extract": {"true"},
"X-Amz-Meta-Minio-Snowball-Prefix": {filepath.Dir(path)},
"Content-Type": {"application/gzip"},
"Content-Encoding": {"gzip"},
})
if err != nil {
return err
}

file, err := os.Open(fsPath)
if err != nil {
return err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return err
}

buf := new(bytes.Buffer)
//g := gzip.NewWriter(buf)
io.Copy(buf, reader)
tarStream := artifacts.NewTarStream()
go func() {
tarStream.Add(filepath.Base(path), file, stat)
tarStream.Close()
}()
io.Copy(buf, tarStream)

req, err := http.NewRequestWithContext(ctx, http.MethodPut, putUrl.String(), buf)
if err != nil {
return err
}
req.ContentLength = int64(buf.Len())
req.Header.Set("X-Amz-Meta-Snowball-Auto-Extract", "true")
req.Header.Set("X-Amz-Meta-Minio-Snowball-Prefix", filepath.Dir(path))
req.Header.Set("Content-Type", "application/gzip")
req.Header.Set("Content-Encoding", "gzip")

req.Header.Set("Content-Type", "application/octet-stream")
//req.Header.Set("Content-Encoding", "gzip")
tr := http.DefaultTransport.(*http.Transport).Clone()
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

Check failure

Code scanning / CodeQL

Disabled TLS certificate check High

InsecureSkipVerify should not be used in production code.
client := &http.Client{Transport: tr}
Expand Down

0 comments on commit a535434

Please sign in to comment.