From b37f52153d8dbbbcc37064bcadf7c771b732913d Mon Sep 17 00:00:00 2001 From: 00pf00 Date: Tue, 28 Nov 2023 14:17:07 +0800 Subject: [PATCH 1/7] fix(gc): Solve the problem that the containerd container is deleted but the assigned IP file is not recycled --- pkg/gc/flannel_gc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/gc/flannel_gc.go b/pkg/gc/flannel_gc.go index 2d662b29..999995c0 100644 --- a/pkg/gc/flannel_gc.go +++ b/pkg/gc/flannel_gc.go @@ -191,8 +191,8 @@ func (gc *flannelGC) shouldCleanup(cid string) bool { if c, err := gc.dockerCli.ContainedInspectContainer(cid); err != nil { if stausErr, ok := status.FromError(err); ok { if stausErr.Code() == codes.NotFound { - //glog.Infof("container %s not found", cid) - return false + glog.Infof("container %s not found", cid) + return true } glog.Warningf("Error inspect container %s: %v", cid, err) } else { From 6787bd67db2354e1443bad1af20fc413fbb556f9 Mon Sep 17 00:00:00 2001 From: 00pf00 Date: Wed, 10 Jan 2024 20:52:05 +0800 Subject: [PATCH 2/7] fix(gc): Use the status of the sandbox to determine whether the container exists --- pkg/api/docker/docker.go | 10 +++++----- pkg/gc/flannel_gc.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/api/docker/docker.go b/pkg/api/docker/docker.go index 2cf4db3d..d2ab46bc 100644 --- a/pkg/api/docker/docker.go +++ b/pkg/api/docker/docker.go @@ -90,15 +90,15 @@ func (d *DockerInterface) DockerInspectContainer(id string) (*dockertypes.Contai return &containerJSON, nil } -func (d *DockerInterface) ContainedInspectContainer(id string) (*criapi.ContainerStatus, error) { +func (d *DockerInterface) ContainedInspectContainer(id string) (*criapi.PodSandboxStatus, error) { ctx, cancel := getTimeoutContext() defer cancel() if os.Getenv("CONTAINERD_HOST") != "" { - request := &criapi.ContainerStatusRequest{ - ContainerId: id, - Verbose: true, + request := &criapi.PodSandboxStatusRequest{ + PodSandboxId: id, + Verbose: true, } - resp, err := d.containerdClient.ContainerStatus(ctx, request) + resp, err := d.containerdClient.PodSandboxStatus(ctx, request) if err != nil { return nil, err } diff --git a/pkg/gc/flannel_gc.go b/pkg/gc/flannel_gc.go index 999995c0..7898195c 100644 --- a/pkg/gc/flannel_gc.go +++ b/pkg/gc/flannel_gc.go @@ -199,7 +199,7 @@ func (gc *flannelGC) shouldCleanup(cid string) bool { glog.Warningf("Error inspect container %s: %v", cid, err) } } else { - if c != nil && (c.State == criapi.ContainerState_CONTAINER_EXITED || c.State == criapi.ContainerState_CONTAINER_UNKNOWN) { + if c != nil && (c.State == criapi.PodSandboxState_SANDBOX_NOTREADY) { glog.Infof("container %s exited %s", c.Id, c.State.String()) return true } From c99f6066cee835b585657a3857a7834ca0c1b048 Mon Sep 17 00:00:00 2001 From: 00pf00 Date: Mon, 15 Jan 2024 16:54:14 +0800 Subject: [PATCH 3/7] fix(workflows): Upgrade go version --- .github/workflows/go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 94af73c8..660b41bf 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -7,10 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.14 + - name: Set up Go 1.18 uses: actions/setup-go@v1 with: - go-version: 1.14.2 + go-version: 1.18.4 id: go - name: Check out code into the Go module directory From 8757e4026892173496da45b24d5176ee337a7a59 Mon Sep 17 00:00:00 2001 From: 00pf00 Date: Mon, 15 Jan 2024 17:07:00 +0800 Subject: [PATCH 4/7] fix(crd): Solve the problem of testing error after crd version upgrade --- pkg/utils/test/test_customresourcedefinition.go | 12 +++++++----- pkg/utils/test/test_helper.go | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/utils/test/test_customresourcedefinition.go b/pkg/utils/test/test_customresourcedefinition.go index 0025b457..3f5f9da2 100644 --- a/pkg/utils/test/test_customresourcedefinition.go +++ b/pkg/utils/test/test_customresourcedefinition.go @@ -75,21 +75,23 @@ func (c *customResourceDefinitionBuilder) Get() *extensionv1.CustomResourceDefin APIVersion: "apiextensions.k8s.io/v1", }, Spec: extensionv1.CustomResourceDefinitionSpec{ - Group: c.group, - Version: c.version, - Scope: "Namespaced", + Group: c.group, + Versions: []extensionv1.CustomResourceDefinitionVersion{{ + Name: c.version, + Subresources: c.subresources, + }}, + Scope: "Namespaced", Names: extensionv1.CustomResourceDefinitionNames{ Plural: plural, Singular: name, Kind: kind, ListKind: kind + "List", }, - Subresources: c.subresources, }, } } // CrdApiVersionAndKind returns the apiVersion and kind of the given CustomResourceDefinition func CrdApiVersionAndKind(crd *extensionv1.CustomResourceDefinition) (string, string) { - return crd.Spec.Group + "/" + crd.Spec.Version, crd.Spec.Names.Kind + return crd.Spec.Group + "/" + crd.Spec.Versions[0].Name, crd.Spec.Names.Kind } diff --git a/pkg/utils/test/test_helper.go b/pkg/utils/test/test_helper.go index f90f1f61..f6d622f8 100644 --- a/pkg/utils/test/test_helper.go +++ b/pkg/utils/test/test_helper.go @@ -51,7 +51,7 @@ func CreateCRDPod(name, namespace string, annotations map[string]string, return Pod().WithName(name).WithNamespace(namespace). WithAnnotations(annotations).WithLabels(map[string]string{"app": appName}). AddOwnerReferences(v1.OwnerReference{ - APIVersion: crd.Spec.Version, + APIVersion: crd.Spec.Versions[0].Name, Kind: crd.Spec.Names.Kind, Name: appName, }). From 1fa33c33caf14eb2d79e7c64556ac2c09c5898d2 Mon Sep 17 00:00:00 2001 From: 00pf00 Date: Mon, 15 Jan 2024 17:19:02 +0800 Subject: [PATCH 5/7] fix(client-go): Solve the problem of testing error after client-go version upgrade --- pkg/ipam/floatingip/ipam_crd_test.go | 3 ++- pkg/ipam/floatingip/store_crd_test.go | 7 ++++--- pkg/ipam/schedulerplugin/bind_test.go | 5 +++-- pkg/ipam/schedulerplugin/crdkey_test.go | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/ipam/floatingip/ipam_crd_test.go b/pkg/ipam/floatingip/ipam_crd_test.go index f27ce34f..f7464d61 100644 --- a/pkg/ipam/floatingip/ipam_crd_test.go +++ b/pkg/ipam/floatingip/ipam_crd_test.go @@ -17,6 +17,7 @@ package floatingip import ( + "context" "encoding/json" "fmt" "net" @@ -142,7 +143,7 @@ func TestAllocateSpecificIP(t *testing.T) { } func checkFIP(ipam *crdIpam, expect ...string) error { - fips, err := ipam.client.GalaxyV1alpha1().FloatingIPs().List(v1.ListOptions{}) + fips, err := ipam.client.GalaxyV1alpha1().FloatingIPs().List(context.Background(), v1.ListOptions{}) if err != nil { return err } diff --git a/pkg/ipam/floatingip/store_crd_test.go b/pkg/ipam/floatingip/store_crd_test.go index 48d24d83..f50d709e 100644 --- a/pkg/ipam/floatingip/store_crd_test.go +++ b/pkg/ipam/floatingip/store_crd_test.go @@ -17,6 +17,7 @@ package floatingip import ( + "context" "fmt" "net" "strings" @@ -44,7 +45,7 @@ func TestAddFloatingIPEventByUser(t *testing.T) { if err := assign(fipCrd, fip); err != nil { t.Fatal(err) } - if _, err := ipam.client.GalaxyV1alpha1().FloatingIPs().Create(fipCrd); err != nil { + if _, err := ipam.client.GalaxyV1alpha1().FloatingIPs().Create(context.Background(), fipCrd, v1.CreateOptions{}); err != nil { t.Fatal(err) } if err := waitFor(ipam, fip.IP, fip.Key, true, node1IPNet.String()); err != nil { @@ -107,7 +108,7 @@ func TestDeleteFloatingIPEvent(t *testing.T) { ip := net.ParseIP(fipCrd.Name) fipCrd.Labels[constant.ReserveFIPLabel] = "" fipCrd.Spec.Key = "pool__reserved-for-node_" - if _, err := ipam.client.GalaxyV1alpha1().FloatingIPs().Create(fipCrd); err != nil { + if _, err := ipam.client.GalaxyV1alpha1().FloatingIPs().Create(context.Background(), fipCrd, v1.CreateOptions{}); err != nil { t.Fatal(err) } if err := waitFor(ipam, ip, fipCrd.Spec.Key, true, node1IPNet.String()); err != nil { @@ -124,7 +125,7 @@ func TestDeleteFloatingIPEvent(t *testing.T) { // test if an event is created by user, deleteFloatingIPEvent should handle it fipCrd.Labels[constant.ReserveFIPLabel] = "" - if err := ipam.client.GalaxyV1alpha1().FloatingIPs().Delete(fipCrd.Name, &v1.DeleteOptions{}); err != nil { + if err := ipam.client.GalaxyV1alpha1().FloatingIPs().Delete(context.Background(), fipCrd.Name, v1.DeleteOptions{}); err != nil { t.Fatal(err) } if err := waitFor(ipam, ip, "", false, node1IPNet.String()); err != nil { diff --git a/pkg/ipam/schedulerplugin/bind_test.go b/pkg/ipam/schedulerplugin/bind_test.go index ea9cd795..39d4a5a3 100644 --- a/pkg/ipam/schedulerplugin/bind_test.go +++ b/pkg/ipam/schedulerplugin/bind_test.go @@ -17,6 +17,7 @@ package schedulerplugin import ( + "context" "encoding/json" "errors" "fmt" @@ -223,7 +224,7 @@ func TestReleaseIPOfFinishedPod(t *testing.T) { t.Fatalf("case %d: %v", i, err) } testCase.updatePodStatus(pod) - if _, err := fipPlugin.Client.CoreV1().Pods(pod.Namespace).UpdateStatus(pod); err != nil { + if _, err := fipPlugin.Client.CoreV1().Pods(pod.Namespace).UpdateStatus(context.Background(), pod, v1.UpdateOptions{}); err != nil { t.Fatalf("case %d: %v", i, err) } if err := wait.Poll(time.Microsecond*10, time.Second*30, func() (done bool, err error) { @@ -285,7 +286,7 @@ func TestFilterBindRequestIPRange(t *testing.T) { t.Fatal(err) } pod.Annotations = cniArgsAnnotation(request) - if _, err := fipPlugin.Client.CoreV1().Pods(pod.Namespace).Update(pod); err != nil { + if _, err := fipPlugin.Client.CoreV1().Pods(pod.Namespace).Update(context.Background(), pod, v1.UpdateOptions{}); err != nil { t.Fatal(err) } // wait for lister updates diff --git a/pkg/ipam/schedulerplugin/crdkey_test.go b/pkg/ipam/schedulerplugin/crdkey_test.go index d12cf200..33e6bd4e 100644 --- a/pkg/ipam/schedulerplugin/crdkey_test.go +++ b/pkg/ipam/schedulerplugin/crdkey_test.go @@ -37,7 +37,7 @@ func TestGetGroupVersionResource(t *testing.T) { if gvr == nil { t.Fatal() } - if gvr.Group != FooCrd.Spec.Group || gvr.Version != FooCrd.Spec.Version || + if gvr.Group != FooCrd.Spec.Group || gvr.Version != FooCrd.Spec.Versions[0].Name || gvr.Resource != FooCrd.Spec.Names.Plural { t.Fatal(gvr) } From bce6365d35ac4ee138ebf6e25fc9d75be819e85e Mon Sep 17 00:00:00 2001 From: 00pf00 Date: Wed, 24 Jan 2024 09:22:31 +0800 Subject: [PATCH 6/7] fix(flannel_gc): When the sandbox is notready, determine whether to clean up the cni file based on the container status in the pod. --- pkg/gc/flannel_gc.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/pkg/gc/flannel_gc.go b/pkg/gc/flannel_gc.go index 7898195c..789d4b5d 100644 --- a/pkg/gc/flannel_gc.go +++ b/pkg/gc/flannel_gc.go @@ -17,11 +17,15 @@ package gc import ( + "context" "flag" "fmt" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "io/ioutil" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "net" "os" "path/filepath" @@ -37,8 +41,10 @@ import ( ) const ( - ContainerExited = "exited" - ContainerDead = "dead" + ContainerExited = "exited" + ContainerDead = "dead" + SandboxName = "io.kubernetes.cri.sandbox-name" + SandboxNamespace = "io.kubernetes.cri.sandbox-namespace" ) var ( @@ -61,16 +67,18 @@ type flannelGC struct { allocatedIPDir []string gcDirs []string dockerCli *docker.DockerInterface + kubeCli kubernetes.Interface quit <-chan struct{} cleanPortFunc func(containerID string) error } -func NewFlannelGC(dockerCli *docker.DockerInterface, quit <-chan struct{}, +func NewFlannelGC(kubeCli kubernetes.Interface, dockerCli *docker.DockerInterface, quit <-chan struct{}, cleanPortFunc func(containerID string) error) GC { dirs := strings.Split(*flagGCDirs, ",") return &flannelGC{ allocatedIPDir: strings.Split(*flagAllocatedIPDir, ","), gcDirs: dirs, + kubeCli: kubeCli, dockerCli: dockerCli, quit: quit, cleanPortFunc: cleanPortFunc, @@ -200,6 +208,19 @@ func (gc *flannelGC) shouldCleanup(cid string) bool { } } else { if c != nil && (c.State == criapi.PodSandboxState_SANDBOX_NOTREADY) { + pod, err := gc.kubeCli.CoreV1().Pods(c.Annotations[SandboxNamespace]).Get(context.Background(), c.Annotations[SandboxName], metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true + } + glog.Errorf("failed to get pod %s", fmt.Sprintf("%s/%s", c.Annotations[SandboxNamespace], c.Annotations[SandboxName])) + return false + } + for _, status := range pod.Status.ContainerStatuses { + if status.State.Waiting != nil || status.State.Running != nil { + return false + } + } glog.Infof("container %s exited %s", c.Id, c.State.String()) return true } From 977b42aa9d2537effe45618ab5ed2690dc470c8a Mon Sep 17 00:00:00 2001 From: 00pf00 Date: Wed, 24 Jan 2024 10:10:09 +0800 Subject: [PATCH 7/7] fix(flannel_gc): Solve compilation errors --- pkg/galaxy/galaxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/galaxy/galaxy.go b/pkg/galaxy/galaxy.go index 19d3a3b6..5a6ed94a 100644 --- a/pkg/galaxy/galaxy.go +++ b/pkg/galaxy/galaxy.go @@ -124,7 +124,7 @@ func (g *Galaxy) Start() error { return err } g.initk8sClient() - gc.NewFlannelGC(g.dockerCli, g.quitChan, g.cleanIPtables).Run() + gc.NewFlannelGC(g.client, g.dockerCli, g.quitChan, g.cleanIPtables).Run() kernel.BridgeNFCallIptables(g.quitChan, g.BridgeNFCallIptables) kernel.IPForward(g.quitChan, g.IPForward) if err := g.setupIPtables(); err != nil {