From e097b3d4b732557ef1d786c86ac66f5e80b14333 Mon Sep 17 00:00:00 2001 From: Jay179-sudo Date: Sat, 29 Jun 2024 22:44:57 +0530 Subject: [PATCH 1/8] Added chainsaw tests for bare pods. Created a test pod and the corresponding clusterrole definition for the test Signed-off-by: Jay179-sudo --- .../.chainsaw-test/chainsaw-test.yaml | 30 +++++++++++++++++++ .../cleanup-bare-pods/.chainsaw-test/pod.yaml | 8 +++++ cleanup/cleanup-bare-pods/clusterrole.yaml | 20 +++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml create mode 100644 cleanup/cleanup-bare-pods/.chainsaw-test/pod.yaml create mode 100644 cleanup/cleanup-bare-pods/clusterrole.yaml diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml new file mode 100644 index 000000000..3c8ac9cb5 --- /dev/null +++ b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml @@ -0,0 +1,30 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: cleanup-bare-pods +spec: + steps: + - name: apply clusterrole + try: + - apply: + file: ../clusterrole.yaml + - name: create a bare pod + try: + - apply: + file: pod.yaml + - assert: + file: pod.yaml + - name: apply cleanup policy + try: + - apply: + file: ../cleanup-bare-pods.yaml + - assert: + file: ../cleanup-bare-pods.yaml + - name: wait for scheduled deletion + try: + - sleep: + duration: 5m30s + - name: check for bare pod + try: + - error: + file: pod.yaml \ No newline at end of file diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/pod.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/pod.yaml new file mode 100644 index 000000000..966df958a --- /dev/null +++ b/cleanup/cleanup-bare-pods/.chainsaw-test/pod.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: bare-pod +spec: + containers: + - name: nginx + image: nginx:1.14.1 diff --git a/cleanup/cleanup-bare-pods/clusterrole.yaml b/cleanup/cleanup-bare-pods/clusterrole.yaml new file mode 100644 index 000000000..9e442827e --- /dev/null +++ b/cleanup/cleanup-bare-pods/clusterrole.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + namespace: kyverno + labels: + app.kubernetes.io/component: cleanup-controller + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno + name: kyverno:cleanup-controller:temp +rules: +- apiGroups: + - "" + resources: + - "pods" + verbs: + - get + - watch + - list + - delete + From 7cb9097cd385338517c5917e627e325febe58fb5 Mon Sep 17 00:00:00 2001 From: Jay179-sudo Date: Tue, 2 Jul 2024 13:16:29 +0530 Subject: [PATCH 2/8] Reduced scheduled time from five minutes to one Signed-off-by: Jay179-sudo --- cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml | 2 +- cleanup/cleanup-bare-pods/cleanup-bare-pods.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml index 3c8ac9cb5..c27a9ba73 100644 --- a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml +++ b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml @@ -23,7 +23,7 @@ spec: - name: wait for scheduled deletion try: - sleep: - duration: 5m30s + duration: 1m30s - name: check for bare pod try: - error: diff --git a/cleanup/cleanup-bare-pods/cleanup-bare-pods.yaml b/cleanup/cleanup-bare-pods/cleanup-bare-pods.yaml index 43804dd08..950b7bfc5 100644 --- a/cleanup/cleanup-bare-pods/cleanup-bare-pods.yaml +++ b/cleanup/cleanup-bare-pods/cleanup-bare-pods.yaml @@ -28,4 +28,4 @@ spec: - key: "{{ target.metadata.ownerReferences[] || `[]` }}" operator: Equals value: [] - schedule: "*/5 * * * *" + schedule: "*/1 * * * *" From 300473c8ab8b452fd46dfb51763eb9ed7efce16f Mon Sep 17 00:00:00 2001 From: Jay179-sudo Date: Tue, 2 Jul 2024 15:53:59 +0530 Subject: [PATCH 3/8] Created a separate test policy referenced by the chainsaw test. Undid changes to the original policy Signed-off-by: Jay179-sudo --- .../.chainsaw-test/chainsaw-test.yaml | 4 +-- .../cleanup-bare-pods-test.yaml | 31 +++++++++++++++++++ .../cleanup-bare-pods/cleanup-bare-pods.yaml | 2 +- 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 cleanup/cleanup-bare-pods/.chainsaw-test/cleanup-bare-pods-test.yaml diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml index c27a9ba73..a5c5ecd63 100644 --- a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml +++ b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml @@ -17,9 +17,9 @@ spec: - name: apply cleanup policy try: - apply: - file: ../cleanup-bare-pods.yaml + file: cleanup-bare-pods-test.yaml - assert: - file: ../cleanup-bare-pods.yaml + file: cleanup-bare-pods-test.yaml - name: wait for scheduled deletion try: - sleep: diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/cleanup-bare-pods-test.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/cleanup-bare-pods-test.yaml new file mode 100644 index 000000000..950b7bfc5 --- /dev/null +++ b/cleanup/cleanup-bare-pods/.chainsaw-test/cleanup-bare-pods-test.yaml @@ -0,0 +1,31 @@ +apiVersion: kyverno.io/v2beta1 +kind: ClusterCleanupPolicy +metadata: + name: clean-bare-pods + annotations: + policies.kyverno.io/title: Cleanup Bare Pods + policies.kyverno.io/category: Other + policies.kyverno.io/severity: medium + policies.kyverno.io/subject: Pod + kyverno.io/kyverno-version: 1.11.1 + policies.kyverno.io/minversion: 1.10.0 + pod-policies.kyverno.io/autogen-controllers: none + kyverno.io/kubernetes-version: "1.27" + policies.kyverno.io/description: >- + A bare Pod is any Pod created directly and not owned by a controller such as a + Deployment or Job. Bare Pods are often create manually by users in an attempt to troubleshoot + an issue. If left in the cluster, they create clutter, increase cost, and can be a security + risk. Bare Pods can be cleaned up periodically through use of a policy. This policy finds + and removes all bare Pods across the cluster. +spec: + match: + any: + - resources: + kinds: + - Pod + conditions: + all: + - key: "{{ target.metadata.ownerReferences[] || `[]` }}" + operator: Equals + value: [] + schedule: "*/1 * * * *" diff --git a/cleanup/cleanup-bare-pods/cleanup-bare-pods.yaml b/cleanup/cleanup-bare-pods/cleanup-bare-pods.yaml index 950b7bfc5..43804dd08 100644 --- a/cleanup/cleanup-bare-pods/cleanup-bare-pods.yaml +++ b/cleanup/cleanup-bare-pods/cleanup-bare-pods.yaml @@ -28,4 +28,4 @@ spec: - key: "{{ target.metadata.ownerReferences[] || `[]` }}" operator: Equals value: [] - schedule: "*/1 * * * *" + schedule: "*/5 * * * *" From bf311b4df630d16cf555387cf978b10e4952e1cf Mon Sep 17 00:00:00 2001 From: Chip Zoller Date: Tue, 2 Jul 2024 07:32:46 -0400 Subject: [PATCH 4/8] Update cleanup/cleanup-bare-pods/clusterrole.yaml Signed-off-by: Chip Zoller --- cleanup/cleanup-bare-pods/clusterrole.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cleanup/cleanup-bare-pods/clusterrole.yaml b/cleanup/cleanup-bare-pods/clusterrole.yaml index 9e442827e..faf779413 100644 --- a/cleanup/cleanup-bare-pods/clusterrole.yaml +++ b/cleanup/cleanup-bare-pods/clusterrole.yaml @@ -11,7 +11,7 @@ rules: - apiGroups: - "" resources: - - "pods" + - pods verbs: - get - watch From e98d9d3e1d589beb18107cfb1c2f5dd89861a05f Mon Sep 17 00:00:00 2001 From: Chip Zoller Date: Tue, 2 Jul 2024 07:33:03 -0400 Subject: [PATCH 5/8] Update cleanup/cleanup-bare-pods/clusterrole.yaml Signed-off-by: Chip Zoller --- cleanup/cleanup-bare-pods/clusterrole.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cleanup/cleanup-bare-pods/clusterrole.yaml b/cleanup/cleanup-bare-pods/clusterrole.yaml index faf779413..6e5bdaf66 100644 --- a/cleanup/cleanup-bare-pods/clusterrole.yaml +++ b/cleanup/cleanup-bare-pods/clusterrole.yaml @@ -6,7 +6,7 @@ metadata: app.kubernetes.io/component: cleanup-controller app.kubernetes.io/instance: kyverno app.kubernetes.io/part-of: kyverno - name: kyverno:cleanup-controller:temp + name: kyverno:cleanup-controller:barepods rules: - apiGroups: - "" From b7f07de1d1bd9488e56ee9f06b7efa13f27088a9 Mon Sep 17 00:00:00 2001 From: Jay179-sudo Date: Tue, 2 Jul 2024 20:44:07 +0530 Subject: [PATCH 6/8] Cleaned up and moved the clusterrole file. Applied a patch to reduce scheduled time Signed-off-by: Jay179-sudo --- .../chainsaw-step-02-assert-1.yaml | 4 +++ .../.chainsaw-test/chainsaw-test.yaml | 14 +++++++-- .../cleanup-bare-pods-test.yaml | 31 ------------------- .../{ => .chainsaw-test}/clusterrole.yaml | 0 4 files changed, 15 insertions(+), 34 deletions(-) create mode 100644 cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-step-02-assert-1.yaml delete mode 100644 cleanup/cleanup-bare-pods/.chainsaw-test/cleanup-bare-pods-test.yaml rename cleanup/cleanup-bare-pods/{ => .chainsaw-test}/clusterrole.yaml (100%) diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-step-02-assert-1.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-step-02-assert-1.yaml new file mode 100644 index 000000000..f0fe23d34 --- /dev/null +++ b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-step-02-assert-1.yaml @@ -0,0 +1,4 @@ +apiVersion: kyverno.io/v2beta1 +kind: ClusterCleanupPolicy +metadata: + name: clean-bare-pods diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml index a5c5ecd63..04fd16d05 100644 --- a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml +++ b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml @@ -7,7 +7,7 @@ spec: - name: apply clusterrole try: - apply: - file: ../clusterrole.yaml + file: clusterrole.yaml - name: create a bare pod try: - apply: @@ -17,9 +17,17 @@ spec: - name: apply cleanup policy try: - apply: - file: cleanup-bare-pods-test.yaml + file: ../cleanup-bare-pods.yaml + - patch: + resource: + apiVersion: kyverno.io/v2beta1 + kind: ClusterCleanupPolicy + metadata: + name: clean-bare-pods + spec: + schedule: "*/1 * * * *" - assert: - file: cleanup-bare-pods-test.yaml + file: chainsaw-step-02-assert-1.yaml - name: wait for scheduled deletion try: - sleep: diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/cleanup-bare-pods-test.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/cleanup-bare-pods-test.yaml deleted file mode 100644 index 950b7bfc5..000000000 --- a/cleanup/cleanup-bare-pods/.chainsaw-test/cleanup-bare-pods-test.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: kyverno.io/v2beta1 -kind: ClusterCleanupPolicy -metadata: - name: clean-bare-pods - annotations: - policies.kyverno.io/title: Cleanup Bare Pods - policies.kyverno.io/category: Other - policies.kyverno.io/severity: medium - policies.kyverno.io/subject: Pod - kyverno.io/kyverno-version: 1.11.1 - policies.kyverno.io/minversion: 1.10.0 - pod-policies.kyverno.io/autogen-controllers: none - kyverno.io/kubernetes-version: "1.27" - policies.kyverno.io/description: >- - A bare Pod is any Pod created directly and not owned by a controller such as a - Deployment or Job. Bare Pods are often create manually by users in an attempt to troubleshoot - an issue. If left in the cluster, they create clutter, increase cost, and can be a security - risk. Bare Pods can be cleaned up periodically through use of a policy. This policy finds - and removes all bare Pods across the cluster. -spec: - match: - any: - - resources: - kinds: - - Pod - conditions: - all: - - key: "{{ target.metadata.ownerReferences[] || `[]` }}" - operator: Equals - value: [] - schedule: "*/1 * * * *" diff --git a/cleanup/cleanup-bare-pods/clusterrole.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/clusterrole.yaml similarity index 100% rename from cleanup/cleanup-bare-pods/clusterrole.yaml rename to cleanup/cleanup-bare-pods/.chainsaw-test/clusterrole.yaml From 2a515a1a97e5c056e17074c917a3c98c4ee6935a Mon Sep 17 00:00:00 2001 From: Jay179-sudo Date: Sat, 6 Jul 2024 00:01:40 +0530 Subject: [PATCH 7/8] fixed file name to cluster-role Signed-off-by: Jay179-sudo --- .../.chainsaw-test/{clusterrole.yaml => cluster-role.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cleanup/cleanup-bare-pods/.chainsaw-test/{clusterrole.yaml => cluster-role.yaml} (100%) diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/clusterrole.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/cluster-role.yaml similarity index 100% rename from cleanup/cleanup-bare-pods/.chainsaw-test/clusterrole.yaml rename to cleanup/cleanup-bare-pods/.chainsaw-test/cluster-role.yaml From 6c943761f46bcf1b40bdc67c322a788b2fad57e4 Mon Sep 17 00:00:00 2001 From: Jay179-sudo Date: Sat, 6 Jul 2024 08:59:27 +0530 Subject: [PATCH 8/8] minor fix Signed-off-by: Jay179-sudo --- cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml index 04fd16d05..d9cf0944a 100644 --- a/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml +++ b/cleanup/cleanup-bare-pods/.chainsaw-test/chainsaw-test.yaml @@ -4,10 +4,10 @@ metadata: name: cleanup-bare-pods spec: steps: - - name: apply clusterrole + - name: apply cluster role try: - apply: - file: clusterrole.yaml + file: cluster-role.yaml - name: create a bare pod try: - apply: