perf(web): serve compressed files from metadata.zip directly

Don't decompress them if the requests's Accept-Encoding header
includes the compression codec specified for the file.

cmd/mrp/webserver.go

@@ -401,13 +401,19 @@ func (self *mrpWebServer) getMetadata(w http.ResponseWriter, req *http.Request)
  http.Error(w, "'..' not allowed in path.", http.StatusBadRequest)
  return
  }
- data, err := self.rt.GetMetadata(pipestance.GetPath(),
- path.Join(p, core.MetadataFilePrefix+name))
+ data, enc, err := self.rt.GetMaybeCompressedMetadata(req.Context(),
+ pipestance.GetPath(),
+ path.Join(p, core.MetadataFilePrefix+name),
+ req.Header.Get("Accept-Encoding"))
  if err != nil {
  http.Error(w, err.Error(), http.StatusNotFound)
  return
  }
  defer data.Close()
+
+ if enc != "" {
+ w.Header().Set("Content-Encoding", enc)
+ }
  api.ServeMetadataFile(w, req, name, data)
 }
 

martian/api/serve_metadata.go

@@ -5,6 +5,7 @@ package api
 import (
  "errors"
  "io"
+ "io/fs"
  "net/http"
  "os"
  "strconv"
@@ -33,7 +34,7 @@ func ServeMetadataFile(w http.ResponseWriter, req *http.Request,
  }
  switch data := data.(type) {
  case interface {
- Stat() (os.FileInfo, error) // TODO(azarchs): Use fs.File, after go 1.15
+ Stat() (fs.FileInfo, error)
  }:
  st, err := data.Stat()
  if err != nil {

martian/core/runtime.go

@@ -575,12 +575,25 @@ func (self *Runtime) GetSerialization(pipestancePath string, name MetadataFileNa
  return nil, false
 }
 
+// Deprecated: use GetMaybeCompressedMetadata instead.
 func (self *Runtime) GetMetadata(pipestancePath string, metadataPath string) (io.ReadCloser, error) {
+ r, _, err := self.GetMaybeCompressedMetadata(context.TODO(), pipestancePath, metadataPath, "")
+ return r, err
+}
+
+// GetMaybeCompressedMetadata returns a reader for the given metadata file.
+// If the metadata is found within a zip archive, and is compressed using a
+// method found in the acceptEncoding string, it is returend along with
+// the encoding.
+func (self *Runtime) GetMaybeCompressedMetadata(ctx context.Context,
+ pipestancePath, metadataPath, acceptEncoding string) (io.ReadCloser, string, error) {
+ defer trace.StartRegion(ctx, "GetMaybeCompressedMetadata").End()
  metadata := NewMetadata("", pipestancePath)
  metadata.loadCache()
  if mdf := MetadataFileName(
  strings.TrimPrefix(metadataPath, MetadataFilePrefix)); metadata.exists(mdf) {
- return metadata.openFile(mdf)
+ f, err := metadata.openFile(mdf)
+ return f, "", err
  }
  if !filepath.IsAbs(metadataPath) {
  metadataPath = path.Join(pipestancePath, metadataPath)
@@ -590,16 +603,17 @@ func (self *Runtime) GetMetadata(pipestancePath string, metadataPath string) (io
 
  // Relative paths outside the pipestance directory will be ignored.
  if !strings.Contains(relPath, "..") {
- if data, err := util.ReadZipFile(metadata.MetadataFilePath(MetadataZip), relPath); err == nil {
- return data, nil
+ if data, m, err := util.ReadZipFileRaw(ctx,
+ metadata.MetadataFilePath(MetadataZip), relPath, acceptEncoding); err == nil {
+ return data, m, nil
  }
  }
  }
  data, err := os.Open(metadataPath)
  if err != nil {
- return nil, err
+ return nil, "", err
  }
- return data, nil
+ return data, "", nil
 }
 
 func (self *Runtime) freeMemMB() int64 {

martian/util/zip.go

@@ -8,10 +8,13 @@ package util
 import (
  "archive/zip"
  "bytes"
+ "context"
  "io"
+ "io/fs"
  "os"
  "path"
  "path/filepath"
+ "runtime/trace"
  "strings"
  "time"
 )
@@ -30,13 +33,16 @@ func readSymlinkInZip(f *zip.File) (string, error) {
 }
 
 // Find a file in the zip. Follows symlinks.
-func findFileInZip(zr *zip.ReadCloser, filePath string) *zip.File {
+func findFileInZip(ctx context.Context, zr *zip.ReadCloser, filePath string) *zip.File {
  for _, f := range zr.File {
+ if ctx.Err() != nil {
+ return nil
+ }
  if f.Mode()&os.ModeSymlink != 0 && strings.HasPrefix(filePath, f.Name+string(os.PathSeparator)) {
  if linkPath, err := readSymlinkInZip(f); err != nil {
  return nil
  } else {
- return findFileInZip(zr, path.Clean(
+ return findFileInZip(ctx, zr, path.Clean(
  path.Join(path.Dir(f.Name), linkPath,
  strings.TrimPrefix(filePath, f.Name+string(os.PathSeparator)))))
  }
@@ -45,7 +51,7 @@ func findFileInZip(zr *zip.ReadCloser, filePath string) *zip.File {
  if linkPath, err := readSymlinkInZip(f); err != nil {
  return f
  } else {
- return findFileInZip(zr, path.Clean(
+ return findFileInZip(ctx, zr, path.Clean(
  path.Join(path.Dir(filePath), linkPath)))
  }
  } else {
@@ -59,13 +65,13 @@ func findFileInZip(zr *zip.ReadCloser, filePath string) *zip.File {
 // Wraps a file within a zip archive, along with the archive itself,
 // as an io.ReadCloser
 type zipFileReader struct {
- zr  *zip.ReadCloser
- file  io.ReadCloser
- modified time.Time
+ zr *zip.ReadCloser
+ file io.ReadCloser
+ info zipFileInfo
 }
 
 func (zr *zipFileReader) ModTime() time.Time {
- return zr.modified
+ return zr.info.ModTime()
 }
 
 func (zr *zipFileReader) Read(p []byte) (int, error) {
@@ -85,11 +91,111 @@ func (zr *zipFileReader) Close() error {
  return err
 }
 
+// fs.FileInfo for a zipFileReader.
+// Extracts just the bits we need so as to avoid holding pointers to the rest of
+// the zip.FileHeader.
+type zipFileInfo struct {
+ modTime time.Time
+ name string
+ size int64
+ mode fs.FileMode
+}
+
+func extractFileInfo(fh *zip.FileHeader, raw bool) zipFileInfo {
+ result := zipFileInfo{
+ name: fh.Name,
+ mode: fh.Mode(),
+ modTime: fh.Modified,
+ }
+ if raw && fh.CompressedSize64 > 0 {
+ result.size = int64(fh.CompressedSize64)
+ } else {
+ result.size = int64(fh.UncompressedSize64)
+ }
+ return result
+}
+
+func (fi zipFileInfo) Name() string { return path.Base(fi.name) }
+func (fi zipFileInfo) Size() int64 { return fi.size }
+func (fi zipFileInfo) IsDir() bool { return fi.Mode().IsDir() }
+func (fi zipFileInfo) ModTime() time.Time { return fi.modTime }
+func (fi zipFileInfo) Mode() fs.FileMode { return fi.mode }
+func (fi zipFileInfo) Type() fs.FileMode { return fi.Mode().Type() }
+func (fi zipFileInfo) Sys() any { return &fi }
+
+func (zr *zipFileReader) Stat() (fs.FileInfo, error) {
+ return &zr.info, nil
+}
+
 // Opens a file within a zip archive for reading.
 func ReadZipFile(zipPath, filePath string) (io.ReadCloser, error) {
+ r, _, err := ReadZipFileRaw(context.TODO(), zipPath, filePath, "")
+ return r, err
+}
+
+func openZipRaw(f *zip.File, accept string) (io.ReadCloser, string, error) {
+ if m := acceptedEncoding(accept, f.Method); m != "" {
+ r, err := f.OpenRaw()
+ if err != nil {
+ return nil, "", err
+ }
+ return io.NopCloser(r), m, err
+ }
+ r, err := f.Open()
+ return r, "", err
+}
+
+// zipMethodToEncoding returns the http-header Content-Encoding name
+// corresponding to the given zip method, or an empty string.
+//
+// Go's standard library only knows how to handle deflate, by default,
+// but for completeness we're adding a few others here, since this code path
+// is intended for use with ZipFile.OpenRaw() where we could in theory
+// be sending compressed data to a web client and letting them deal with
+// decompression.
+func zipMethodToEncoding(method uint16) string {
+ // For methods other than deflate, see zip APPNOTE.txt section 4.4.5
+ switch method {
+ case zip.Deflate:
+ return "deflate"
+ case 12:
+ return "bz2"
+ case 93:
+ return "zstd"
+ case 95:
+ return "xz"
+ }
+ return ""
+}
+
+// acceptedEncoding returns the encoding name corresponding to the zip
+// compression method, if it is present in accept.
+func acceptedEncoding(accept string, method uint16) string {
+ m := zipMethodToEncoding(method)
+ // strings.Contains isn't a formally correct way to test whether the
+ // encoding is accepted, however there's no case where an encoding
+ // we support exists as a substring of a different valid encoding specifier,
+ // (ignoring `;q=` modifiers, which we don't care about because we aren't
+ // actually giving the user choices here) so this is correct in practice,
+ // and more computationally efficient than a test that would be robust
+ // against invalid specifiers in an `Accept-Encoding` header.
+ if m != "" && !strings.Contains(accept, m) {
+ m = ""
+ }
+ return m
+}
+
+// ReadZipRaw opens a file within an archive for reading, without decompressing.
+//
+// The second return argument specifies the encoding found within the file,
+// generally either the empty string (uncompressed) or deflate.
+func ReadZipFileRaw(ctx context.Context,
+ zipPath, filePath,
+ acceptEncoding string) (io.ReadCloser, string, error) {
+ defer trace.StartRegion(ctx, "ReadZipFileRaw").End()
  zr, err := zip.OpenReader(zipPath)
  if err != nil {
- return nil, err
+ return nil, "", err
  }
  found := false
  defer func() {
@@ -98,16 +204,26 @@ func ReadZipFile(zipPath, filePath string) (io.ReadCloser, error) {
  }
  }()
 
- if f := findFileInZip(zr, filePath); f != nil {
- in, err := f.Open()
+ if err := ctx.Err(); err != nil {
+ return nil, "", err
+ }
+
+ if f := findFileInZip(ctx, zr, filePath); f != nil {
+ in, m, err := openZipRaw(f, acceptEncoding)
  if err != nil {
- return nil, err
+ return nil, "", err
  }
  found = true
- return &zipFileReader{zr: zr, file: in, modified: f.Modified}, nil
+ return &zipFileReader{
+ zr: zr, file: in,
+ info: extractFileInfo(&f.FileHeader, m != ""),
+ }, m, nil
+ }
+ if err := ctx.Err(); err != nil {
+ return nil, "", err
  }
 
- return nil, &ZipError{zipPath, filePath}
+ return nil, "", &ZipError{zipPath, filePath}
 }
 
 func ReadZip(zipPath string, filePath string) ([]byte, error) {
@@ -117,7 +233,7 @@ func ReadZip(zipPath string, filePath string) ([]byte, error) {
  }
  defer zr.Close()
 
- if f := findFileInZip(zr, filePath); f != nil {
+ if f := findFileInZip(context.TODO(), zr, filePath); f != nil {
  in, err := f.Open()
  if err != nil {
  return nil, err