Merge pull request #453 from changweige/mounts-empty-parent

Support using nydus-snapshotter as Builkdit's oci-woker-snapshotter to gain lazyload when building images

pkg/filesystem/fs.go

@@ -185,10 +185,17 @@ func (fs *Filesystem) Mount(snapshotID string, labels map[string]string) (err er
  return nil
  }
 
+ var imageID string
  imageID, ok := labels[snpkg.TargetRefLabel]
  if !ok {
- return errors.Errorf("failed to find image ref of snapshot %s, labels %v",
- snapshotID, labels)
+ // FIXME: Buildkit does not pass labels defined in containerd‘s fashion. So
+ // we have to use stargz snapshotter specific labels until Buildkit generalize it the necessary
+ // labels for all remote snapshotters.
+ imageID, ok = labels["containerd.io/snapshot/remote/stargz.reference"]
+ if !ok {
+ return errors.Errorf("failed to find image ref of snapshot %s, labels %v",
+ snapshotID, labels)
+ }
  }
 
  r := daemon.RafsSet.Get(snapshotID)
@@ -330,7 +337,7 @@ func (fs *Filesystem) Umount(ctx context.Context, snapshotID string) error {
 func (fs *Filesystem) CacheUsage(ctx context.Context, blobDigest string) (snapshots.Usage, error) {
  digest := digest.Digest(blobDigest)
  if err := digest.Validate(); err != nil {
- return snapshots.Usage{}, errors.Wrapf(err, "invalid blob digest from label %s, digest=%s",
+ return snapshots.Usage{}, errors.Wrapf(err, "invalid blob digest from label %q, digest=%s",
  snpkg.TargetLayerDigestLabel, blobDigest)
  }
  blobID := digest.Hex()
@@ -340,7 +347,7 @@ func (fs *Filesystem) CacheUsage(ctx context.Context, blobDigest string) (snapsh
 func (fs *Filesystem) RemoveCache(blobDigest string) error {
  digest := digest.Digest(blobDigest)
  if err := digest.Validate(); err != nil {
- return errors.Wrapf(err, "invalid blob digest from label %s. digest=%s",
+ return errors.Wrapf(err, "invalid blob digest from label %q, digest=%s",
  snpkg.TargetLayerDigestLabel, blobDigest)
  }
  blobID := digest.Hex()

pkg/referrer/manager.go

@@ -57,7 +57,7 @@ func (manager *Manager) CheckReferrer(ctx context.Context, ref string, manifestD
  return nil, errors.Wrap(err, "check referrer")
  }
 
- // FIXME: how invalidating the LRU cache if referrers update?
+ // FIXME: how to invalidate the LRU cache if referrers update?
  manager.cache.Add(manifestDigest, *metaLayer)
 
  return metaLayer, nil

pkg/referrer/referrer.go

@@ -34,7 +34,7 @@ func newReferrer(keyChain *auth.PassKeyChain, insecure bool) *referrer {
  }
 }
 
-// checkReferrer fetches the referrers and parse out the nydus
+// checkReferrer fetches the referrers and parses out the nydus
 // image by specified manifest digest.
 // it's using distribution list referrers API.
 func (r *referrer) checkReferrer(ctx context.Context, ref string, manifestDigest digest.Digest) (*ocispec.Descriptor, error) {

pkg/snapshot/storage.go

@@ -68,7 +68,7 @@ func IterateParentSnapshots(ctx context.Context, ms *storage.MetaStore, key stri
 
  defer func() {
  if err := t.Rollback(); err != nil {
- log.L.WithError(err).Errorf("Rollback traction %s", key)
+ log.L.WithError(err).Errorf("Rollback transaction %s", key)
  }
  }()
 

snapshot/process.go

@@ -92,13 +92,15 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry,
  handler = remoteHandler(id, info.Labels)
  }
 
- if id, info, err := sn.findReferrerLayer(ctx, key); err == nil {
- logger.Infof("found referenced nydus manifest for image: %s", info.Labels[snpkg.TargetRefLabel])
- metaPath := path.Join(sn.snapshotDir(id), "fs", "image.boot")
- if err := sn.fs.TryFetchMetadata(ctx, info.Labels, metaPath); err != nil {
- return nil, "", errors.Wrap(err, "try fetch metadata")
+ if sn.fs.ReferrerDetectEnabled() {
+ if id, info, err := sn.findReferrerLayer(ctx, key); err == nil {
+ logger.Infof("found referenced nydus manifest for image: %s", info.Labels[snpkg.TargetRefLabel])
+ metaPath := path.Join(sn.snapshotDir(id), "fs", "image.boot")
+ if err := sn.fs.TryFetchMetadata(ctx, info.Labels, metaPath); err != nil {
+ return nil, "", errors.Wrap(err, "try fetch metadata")
+ }
+ handler = remoteHandler(id, info.Labels)
  }
- handler = remoteHandler(id, info.Labels)
  }
 
  if sn.fs.StargzEnabled() {

snapshot/snapshot.go

@@ -286,39 +286,51 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er
 
  id, info, _, err := snapshot.GetSnapshotInfo(ctx, o.ms, key)
  if err != nil {
- return nil, errors.Wrapf(err, "get snapshot %s info", key)
+ return nil, errors.Wrapf(err, "mounts get snapshot %q info", key)
  }
  log.L.Infof("[Mounts] snapshot %s ID %s Kind %s", key, id, info.Kind)
 
  if label.IsNydusMetaLayer(info.Labels) {
  err = o.fs.WaitUntilReady(id)
  if err != nil {
- return nil, errors.Wrapf(err, "mounts: snapshot %s is not ready, err: %v", id, err)
+ // Skip waiting if clients is unpacking nydus artifacts to `mounts`
+ // For example, nydus-snapshotter's client like Buildkit is calling snapshotter in below workflow:
+ // 1. [Prepare] snapshot for the uppermost layer - bootstrap
+ // 2. [Mounts]
+ // 3. Unpacking by applying the mounts, then we get bootstrap in its path position.
+ // In above steps, no container write layer is called to set up from nydus-snapshotter. So it has no
+ // chance to start nydusd, during which the Rafs instance is created.
+ if !errors.Is(err, errdefs.ErrNotFound) {
+ return nil, errors.Wrapf(err, "mounts: snapshot %s is not ready, err: %v", id, err)
+ }
+ } else {
+ needRemoteMounts = true
+ metaSnapshotID = id
  }
- needRemoteMounts = true
- metaSnapshotID = id
  }
 
  if info.Kind == snapshots.KindActive {
  pKey := info.Parent
- pID, info, _, err := snapshot.GetSnapshotInfo(ctx, o.ms, pKey)
- if err != nil {
- return nil, errors.Wrapf(err, "get snapshot %s info", pKey)
- }
-
- if label.IsNydusMetaLayer(info.Labels) {
- err = o.fs.WaitUntilReady(pID)
- if err != nil {
- return nil, errors.Wrapf(err, "mounts: snapshot %s is not ready, err: %v", pID, err)
+ if pID, info, _, err := snapshot.GetSnapshotInfo(ctx, o.ms, pKey); err == nil {
+ if label.IsNydusMetaLayer(info.Labels) {
+ if err = o.fs.WaitUntilReady(pID); err != nil {
+ return nil, errors.Wrapf(err, "mounts: snapshot %s is not ready, err: %v", pID, err)
+ }
+ metaSnapshotID = pID
+ needRemoteMounts = true
+ }
+ } else {
+ if !errors.Is(err, errdefs.ErrNotFound) {
+ return nil, errors.Wrapf(err, "get parent snapshot info, parent key=%q", pKey)
  }
- metaSnapshotID = pID
- needRemoteMounts = true
  }
  }
 
- if id, _, err := o.findReferrerLayer(ctx, key); err == nil {
- needRemoteMounts = true
- metaSnapshotID = id
+ if o.fs.ReferrerDetectEnabled() {
+ if id, _, err := o.findReferrerLayer(ctx, key); err == nil {
+ needRemoteMounts = true
+ metaSnapshotID = id
+ }
  }
 
  snap, err := snapshot.GetSnapshot(ctx, o.ms, key)
@@ -349,7 +361,7 @@ func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s
  return nil, err
  }
 
- logger.Debugf("prepare snapshot with labels %v", info.Labels)
+ logger.Debugf("[Prepare] snapshot with labels %v", info.Labels)
 
  processor, target, err := chooseProcessor(ctx, logger, o, s, key, parent, info.Labels, func() string { return o.upperPath(s.ID) })
  if err != nil {
@@ -451,7 +463,7 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap
  return err
  }
 
- log.L.Infof("[Commit] snapshot with key %s snapshot id %s", key, id)
+ log.L.Infof("[Commit] snapshot with key %q snapshot id %s", key, id)
 
  var usage fs.Usage
  // For OCI compatibility, we calculate disk usage and commit the usage to DB.