feat: add nginx for kubelet

hack/k8s-in-k8s/generate_env.sh

@@ -80,10 +80,18 @@ function GetFileName() {
 
 function GetDirectory() {
  local fullpath="$1"
+ if [ -z "$fullpath" ]; then
+ echo "Error: No directory found."
+ exit 1
+ fi
  local directory=$(dirname "$fullpath")
  echo "$directory"
 }
 
+function GetMasterNodeIPs() {
+ kubectl get nodes -l node-role.kubernetes.io/master="" -o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{" "}{end}'
+}
+
 # kubelet config name
 KUBELET_CONFIG_NAME=$(GetFileName "$(GetKubeletConfigFilePath)")
 # path for kubelet 
@@ -97,6 +105,15 @@ PATH_KUBERNETES_PKI=$(GetDirectory "$(GetKubernetesCaPath)")
 PATH_KUBERNETES=$(GetDirectory $PATH_KUBERNETES_PKI)
 HOST_CORE_DNS=$(GetKubeDnsClusterIP)
 
+DOCKER_IMAGE_NGINX="registry.paas/cmss/nginx:1.21.4"
+SERVERS=$(GetMasterNodeIPs)
+if [ -z "$SERVERS" ]; then
+ echo "Error: No master nodes found or failed to retrieve node IPs."
+ exit 1
+fi
+LOCAL_PORT="6443"
+LOCAL_IP="127.0.0.1" # [::1]
+
 echo "#!/usr/bin/env bash
 
 # #####
@@ -128,6 +145,13 @@ USE_KUBEADM=false
 # Generate kubelet.conf TIMEOUT
 KUBELET_CONF_TIMEOUT=30
 
+# load balance
+DOCKER_IMAGE_NGINX=$DOCKER_IMAGE_NGINX
+SERVERS=($SERVERS)
+LOCAL_PORT="6443"
+LOCAL_IP="127.0.0.1" # [::1]
+USE_NGINX=true
+
 function GenerateKubeadmConfig() {
  echo \"---
 apiVersion: kubeadm.k8s.io/v1beta2
@@ -147,6 +171,10 @@ nodeRegistration:
 }
 
 function GenerateStaticNginxProxy() {
+ config_path=/apps/conf/nginx
+ if [ "\$1" == \"true\" ]; then
+ config_path=\$PATH_FILE_TMP
+ fi
  echo \"apiVersion: v1
 kind: Pod
 metadata:
@@ -155,7 +183,7 @@ metadata:
  namespace: kube-system
 spec:
  containers:
- - image: registry.paas/cmss/nginx:1.21.4
+ - image: \$DOCKER_IMAGE_NGINX
  imagePullPolicy: IfNotPresent
  name: nginx-proxy
  resources:
@@ -175,7 +203,7 @@ spec:
  priorityClassName: system-node-critical
  volumes:
  - hostPath:
- path: /apps/conf/nginx
+ path: \$config_path
  type: 
  name: etc-nginx
 status: {}\" > $PATH_KUBERNETES/manifests/nginx-proxy.yaml

hack/k8s-in-k8s/kubelet_node_helper.sh

@@ -330,11 +330,103 @@ function version() {
  echo "$SCRIPT_VERSION"
 }
 
+
+function is_ipv6() {
+ if [[ "$1" =~ : ]]; then
+ return 0 
+ else
+ return 1
+ fi
+}
+
+function install_lb() {
+ if [ -z "$USE_NGINX" ]; then
+ export USE_KUBEADM=false
+ fi
+
+ if [ "$USE_NGINX" = false ]; then
+ exit 0
+ fi
+
+ echo "exec(1/6): get port of apiserver...."
+
+ PORT=$(grep 'server:' "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" | awk -F '[:/]' '{print $NF}')
+
+ if [ -z "$PORT" ]; then
+ echo "can not get port"
+ exit 1
+ else
+ echo "port is $PORT"
+ fi
+
+ if [ "$LOCAL_PORT" -eq "$PORT" ]; then
+ echo "Error: LOCAL_PORT ($LOCAL_PORT) cannot be the same as the backend port ($PORT)."
+ exit 0
+ fi
+
+ # Start generating nginx.conf
+ echo "exec(1/6): generate nginx.conf...."
+ cat <<EOL > "$PATH_FILE_TMP/nginx.conf"
+error_log stderr notice;
+worker_processes 1;
+events {
+ multi_accept on;
+ use epoll;
+ worker_connections 1024;
+}
+
+stream {
+ upstream kube_apiserver {
+ least_conn;
+EOL
+
+ # Loop through the array and append each server to the nginx.conf file
+ for SERVER in "${SERVERS[@]}"; do
+ if is_ipv6 "$SERVER"; then
+ echo " server [$SERVER]:$PORT;" >> "$PATH_FILE_TMP/nginx.conf"
+ else
+ echo " server $SERVER:$PORT;" >> "$PATH_FILE_TMP/nginx.conf"
+ fi
+ done
+
+ # Continue writing the rest of the nginx.conf
+ cat <<EOL >> "$PATH_FILE_TMP/nginx.conf"
+ }
+ server {
+ listen [::]:$LOCAL_PORT;
+ listen 6443;
+ proxy_pass kube_apiserver;
+ proxy_timeout 10m;
+ proxy_connect_timeout 10s;
+ }
+}
+EOL
+
+ echo "exec(1/6): create static pod"
+ GenerateStaticNginxProxy true
+
+
+ echo "exec(1/6): restart static pod"
+ mv "${PATH_KUBERNETES}/manifests/nginx-proxy.yaml" "${PATH_KUBERNETES}/nginx-proxy.yaml"
+ sleep 2
+ mv "${PATH_KUBERNETES}/nginx-proxy.yaml" "${PATH_KUBERNETES}/manifests/nginx-proxy.yaml"
+
+ echo "exec(1/6): update kubelet.conf"
+ cp "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}.bak"
+ sed -i "s|server: .*|server: https://${LOCAL_IP}:${LOCAL_PORT}|" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}"
+
+ echo "exec(1/6): restart kubelet"
+ systemctl restart kubelet
+}
+
 # See how we were called.
 case "$1" in
  unjoin)
  unjoin
  ;;
+ install_lb)
+ install_lb
+ ;;
  join)
  join
  ;;

pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go

@@ -336,6 +336,26 @@ func NewWaitNodeReadyTask(isHost bool) Task {
  }
 }
 
+func NewInstallNginxTask() Task {
+ return Task{
+ Name: "remote install nginx",
+ Retry: true,
+ Run: func(ctx context.Context, to TaskOpt, _ interface{}) (interface{}, error) {
+ exectHelper := exector.NewExectorHelper(to.NodeInfo.Spec.NodeIP, "")
+
+ joinCmd := &exector.CMDExector{
+ Cmd: fmt.Sprintf("bash %s install_lb", env.GetExectorShellName()),
+ }
+ to.Loger().Infof("install nginx %s with cmd: %s", to.NodeInfo.Name, joinCmd.Cmd)
+ ret := exectHelper.DoExector(ctx.Done(), joinCmd)
+ if ret.Status != exector.SUCCESS {
+ return nil, fmt.Errorf("nstall nginx %s failed: %s", to.NodeInfo.Name, ret.String())
+ }
+ return nil, nil
+ },
+ }
+}
+
 // nolint:dupl
 func NewUpdateVirtualNodeLabelsTask() Task {
  return Task{

pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go

@@ -90,6 +90,7 @@ func NewJoinWorkFlow() WorkflowData {
  task.NewRemoteUpdateConfigYamlTask(),
  task.NewRemoteNodeJoinTask(),
  task.NewWaitNodeReadyTask(false),
+ task.NewInstallNginxTask(),
  task.NewUpdateVirtualNodeLabelsTask(),
  task.NewUpdateNodePoolItemStatusTask(v1alpha1.NodeInUse, false),
  }

pkg/kubenest/tasks/manifests_components.go

@@ -106,7 +106,6 @@ func applyComponentsManifests(r workflow.RunData) error {
  if nodeCount < constants.VipKeepAlivedReplicas {
  keepalivedReplicas = int(nodeCount)
  }
-
  templatedMapping["KeepalivedReplicas"] = keepalivedReplicas
  }
 
@@ -124,6 +123,10 @@ func applyComponentsManifests(r workflow.RunData) error {
  },
  })
 
+ for k, v := range data.PluginOptions() {
+ templatedMapping[k] = v
+ }
+
  for _, component := range components {
  klog.V(2).Infof("Deploy component %s", component.Name)
  if v, ok := skipComponents[component.Name]; ok && v {