From 455dffcf43a99495f563627650e2a3f2fe96a1c2 Mon Sep 17 00:00:00 2001 From: Anushka Mittal Date: Tue, 27 Aug 2024 16:52:06 +0530 Subject: [PATCH 1/4] NDEV-19864: deferred loading of images Signed-off-by: Anushka Mittal --- pkg/engine/context/context.go | 63 ++++++++++++++++++++++++------ pkg/engine/context/context_test.go | 59 +++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 14 deletions(-) diff --git a/pkg/engine/context/context.go b/pkg/engine/context/context.go index 7e167a05cd7a..8baedb36ea2f 100644 --- a/pkg/engine/context/context.go +++ b/pkg/engine/context/context.go @@ -296,21 +296,23 @@ func (ctx *context) AddImageInfo(info apiutils.ImageInfo, cfg config.Configurati } func (ctx *context) AddImageInfos(resource *unstructured.Unstructured, cfg config.Configuration) error { - images, err := apiutils.ExtractImagesFromResource(*resource, nil, cfg) - if err != nil { - return err - } - if len(images) == 0 { - return nil - } - ctx.images = images - utm, err := convertImagesToUntyped(images) - if err != nil { - return err + imageInfoLoader := &ImageInfoLoader{ + resource: resource, + eCtx: ctx, + cfg: cfg, } + dl, err := NewDeferredLoader("images", imageInfoLoader, logger) - logging.V(4).Info("updated image info", "images", utm) - return addToContext(ctx, utm, "images") + if toggle.FromContext(cont.Background()).EnableDeferredLoading() { + if err := ctx.AddDeferredLoader(dl); err != nil { + return err + } + } else { + if err := imageInfoLoader.LoadData(); err != nil { + return err + } + } + return nil } func convertImagesToUntyped(images map[string]map[string]apiutils.ImageInfo) (map[string]interface{}, error) { @@ -335,6 +337,35 @@ func convertImagesToUntyped(images map[string]map[string]apiutils.ImageInfo) (ma return results, nil } +type ImageInfoLoader struct { + resource *unstructured.Unstructured + hasLoaded bool + eCtx *context + cfg config.Configuration +} + +func (l *ImageInfoLoader) HasLoaded() bool { + return l.hasLoaded +} + +func (l *ImageInfoLoader) LoadData() error { + images, err := apiutils.ExtractImagesFromResource(*resource, nil, cfg) + if err != nil { + return err + } + if len(images) == 0 { + return nil + } + ctx.images = images + utm, err := convertImagesToUntyped(images) + if err != nil { + return err + } + + logging.V(4).Info("updated image info", "images", utm) + return addToContext(ctx, utm, "images") +} + func (ctx *context) GenerateCustomImageInfo(resource *unstructured.Unstructured, imageExtractorConfigs kyvernov1.ImageExtractorConfigs, cfg config.Configuration) (map[string]map[string]apiutils.ImageInfo, error) { images, err := apiutils.ExtractImagesFromResource(*resource, imageExtractorConfigs, cfg) if err != nil { @@ -350,6 +381,12 @@ func (ctx *context) GenerateCustomImageInfo(resource *unstructured.Unstructured, } func (ctx *context) ImageInfo() map[string]map[string]apiutils.ImageInfo { + // force load of image info from deferred loader + if len(ctx.images) == 0 { + if err := ctx.loadDeferred("images"); err != nil { + return map[string]map[string]apiutils.ImageInfo{} + } + } return ctx.images } diff --git a/pkg/engine/context/context_test.go b/pkg/engine/context/context_test.go index d24f4980760c..5920efbab9fa 100644 --- a/pkg/engine/context/context_test.go +++ b/pkg/engine/context/context_test.go @@ -8,9 +8,13 @@ import ( "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/engine/jmespath" authenticationv1 "k8s.io/api/authentication/v1" + kubeutils "github.com/kyverno/kyverno/pkg/utils/kube" ) -var jp = jmespath.New(config.NewDefaultConfiguration(false)) +var ( + jp = jmespath.New(config.NewDefaultConfiguration(false)) + cfg = config.NewDefaultConfiguration(false) +) func Test_addResourceAndUserContext(t *testing.T) { var err error @@ -123,3 +127,56 @@ func Test_addResourceAndUserContext(t *testing.T) { t.Error("expected result does not match") } } + +func Test_ImageInfoLoader(t *testing.T) { + resource1, err := kubeutils.BytesToUnstructured([]byte(`{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "test-pod", + "namespace": "default" + }, + "spec": { + "containers": [{ + "name": "test_container", + "image": "nginx:latest" + }] + } + }`)) + assert.Nil(t, err) + newctx := newContext() + err = newctx.AddImageInfos(resource1, cfg) + assert.Nil(t, err) + // images not loaded + assert.Nil(t, newctx.images) + // images loaded on Query + name, err := newctx.Query("images.containers.test_container.name") + assert.Nil(t, err) + assert.Equal(t, name, "nginx") +} + +func Test_ImageInfoLoader_OnDirectCall(t *testing.T) { + resource1, err := kubeutils.BytesToUnstructured([]byte(`{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "test-pod", + "namespace": "default" + }, + "spec": { + "containers": [{ + "name": "test_container", + "image": "nginx:latest" + }] + } + }`)) + assert.Nil(t, err) + newctx := newContext() + err = newctx.AddImageInfos(resource1, cfg) + assert.Nil(t, err) + // images not loaded + assert.Nil(t, newctx.images) + // images loaded on explicit call to ImageInfo + imageinfos := newctx.ImageInfo() + assert.Equal(t, imageinfos["containers"]["test_container"].Name, "nginx") +} From 6e20372d7fc3034ee7a000e6aeb7ac7cde3fda57 Mon Sep 17 00:00:00 2001 From: Anushka Mittal Date: Tue, 27 Aug 2024 18:33:56 +0530 Subject: [PATCH 2/4] NDEV-19864: add correct packages Signed-off-by: Anushka Mittal --- pkg/engine/context/context.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/engine/context/context.go b/pkg/engine/context/context.go index 8baedb36ea2f..424336a9c0d2 100644 --- a/pkg/engine/context/context.go +++ b/pkg/engine/context/context.go @@ -1,10 +1,13 @@ package context import ( + cont "context" "fmt" "regexp" "strings" + "github.com/kyverno/kyverno/pkg/toggle" + jsoniter "github.com/json-iterator/go" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" kyvernov1beta1 "github.com/kyverno/kyverno/api/kyverno/v1beta1" @@ -302,7 +305,9 @@ func (ctx *context) AddImageInfos(resource *unstructured.Unstructured, cfg confi cfg: cfg, } dl, err := NewDeferredLoader("images", imageInfoLoader, logger) - + if err != nil { + return err + } if toggle.FromContext(cont.Background()).EnableDeferredLoading() { if err := ctx.AddDeferredLoader(dl); err != nil { return err @@ -381,12 +386,12 @@ func (ctx *context) GenerateCustomImageInfo(resource *unstructured.Unstructured, } func (ctx *context) ImageInfo() map[string]map[string]apiutils.ImageInfo { - // force load of image info from deferred loader - if len(ctx.images) == 0 { - if err := ctx.loadDeferred("images"); err != nil { - return map[string]map[string]apiutils.ImageInfo{} - } + // force load of image info from deferred loader + if len(ctx.images) == 0 { + if err := ctx.loadDeferred("images"); err != nil { + return map[string]map[string]apiutils.ImageInfo{} } + } return ctx.images } From 71b3b9e05b61849f566f71d48e2b738b2a71f3ba Mon Sep 17 00:00:00 2001 From: Anushka Mittal Date: Sat, 31 Aug 2024 11:00:09 +0530 Subject: [PATCH 3/4] NDEV-19864: initial changes for deferred loading in 1.10 Signed-off-by: Anushka Mittal --- pkg/engine/context/context.go | 9 ++++----- pkg/engine/context/context_test.go | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/engine/context/context.go b/pkg/engine/context/context.go index 424336a9c0d2..d5342d720fbc 100644 --- a/pkg/engine/context/context.go +++ b/pkg/engine/context/context.go @@ -1,7 +1,6 @@ package context import ( - cont "context" "fmt" "regexp" "strings" @@ -308,7 +307,7 @@ func (ctx *context) AddImageInfos(resource *unstructured.Unstructured, cfg confi if err != nil { return err } - if toggle.FromContext(cont.Background()).EnableDeferredLoading() { + if toggle.EnableDeferredLoading.Enabled() { if err := ctx.AddDeferredLoader(dl); err != nil { return err } @@ -354,21 +353,21 @@ func (l *ImageInfoLoader) HasLoaded() bool { } func (l *ImageInfoLoader) LoadData() error { - images, err := apiutils.ExtractImagesFromResource(*resource, nil, cfg) + images, err := apiutils.ExtractImagesFromResource(*l.resource, nil, l.cfg) if err != nil { return err } if len(images) == 0 { return nil } - ctx.images = images + l.eCtx.images = images utm, err := convertImagesToUntyped(images) if err != nil { return err } logging.V(4).Info("updated image info", "images", utm) - return addToContext(ctx, utm, "images") + return addToContext(l.eCtx, utm, "images") } func (ctx *context) GenerateCustomImageInfo(resource *unstructured.Unstructured, imageExtractorConfigs kyvernov1.ImageExtractorConfigs, cfg config.Configuration) (map[string]map[string]apiutils.ImageInfo, error) { diff --git a/pkg/engine/context/context_test.go b/pkg/engine/context/context_test.go index 5920efbab9fa..6a32ecb4c21d 100644 --- a/pkg/engine/context/context_test.go +++ b/pkg/engine/context/context_test.go @@ -7,8 +7,9 @@ import ( urkyverno "github.com/kyverno/kyverno/api/kyverno/v1beta1" "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/engine/jmespath" - authenticationv1 "k8s.io/api/authentication/v1" kubeutils "github.com/kyverno/kyverno/pkg/utils/kube" + "github.com/stretchr/testify/assert" + authenticationv1 "k8s.io/api/authentication/v1" ) var ( From 705b1326fe5dcecc166f5ffc3512b5fc61f8aefe Mon Sep 17 00:00:00 2001 From: Anushka Mittal Date: Tue, 3 Sep 2024 10:42:42 +0530 Subject: [PATCH 4/4] NDEV-19864: lint error Signed-off-by: Anushka Mittal --- pkg/engine/context/context.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/engine/context/context.go b/pkg/engine/context/context.go index d5342d720fbc..1490bcb7d694 100644 --- a/pkg/engine/context/context.go +++ b/pkg/engine/context/context.go @@ -5,8 +5,6 @@ import ( "regexp" "strings" - "github.com/kyverno/kyverno/pkg/toggle" - jsoniter "github.com/json-iterator/go" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" kyvernov1beta1 "github.com/kyverno/kyverno/api/kyverno/v1beta1" @@ -14,6 +12,7 @@ import ( "github.com/kyverno/kyverno/pkg/engine/jmespath" "github.com/kyverno/kyverno/pkg/engine/jsonutils" "github.com/kyverno/kyverno/pkg/logging" + "github.com/kyverno/kyverno/pkg/toggle" apiutils "github.com/kyverno/kyverno/pkg/utils/api" admissionv1 "k8s.io/api/admission/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"