diff --git a/charts/aws-efs-csi-driver/templates/_node.tpl b/charts/aws-efs-csi-driver/templates/_node.tpl new file mode 100644 index 000000000..177d0b0dc --- /dev/null +++ b/charts/aws-efs-csi-driver/templates/_node.tpl @@ -0,0 +1,184 @@ +# Node Service +{{- define "node" }} +{{- if ne .Values.node.enable false }} +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .NodeName }} + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +spec: + selector: + matchLabels: + app: {{ .NodeName }} + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.node.updateStrategy }} + updateStrategy: + {{ toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + app: {{ .NodeName }} + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- with .Values.node.hostAliases }} + hostAliases: + {{- range $k, $v := . }} + - ip: {{ $v.ip }} + hostnames: + - {{ $k }}.efs.{{ $v.region }}.amazonaws.com + {{- end }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + hostNetwork: true + dnsPolicy: {{ .Values.node.dnsPolicy }} + {{- with .Values.node.dnsConfig }} + dnsConfig: {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: system-node-critical + {{- with .Values.node.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.node.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: efs-plugin + securityContext: + privileged: true + image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v={{ .Values.node.logLevel }} + - --vol-metrics-opt-in={{ hasKey .Values.node "volMetricsOptIn" | ternary .Values.node.volMetricsOptIn false }} + - --vol-metrics-refresh-period={{ hasKey .Values.node "volMetricsRefreshPeriod" | ternary .Values.node.volMetricsRefreshPeriod 240 }} + - --vol-metrics-fs-rate-limit={{ hasKey .Values.node "volMetricsFsRateLimit" | ternary .Values.node.volMetricsFsRateLimit 5 }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + {{- if .Values.useFIPS }} + - name: AWS_USE_FIPS_ENDPOINT + value: "true" + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: {{ .Values.node.kubelet | default "/var/lib/kubelet" }} + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: efs-state-dir + mountPath: /var/run/efs + - name: efs-utils-config + mountPath: /var/amazon/efs + - name: efs-utils-config-legacy + mountPath: /etc/amazon/efs-legacy + ports: + - name: healthz + containerPort: {{ .Values.node.healthPort }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.node.logLevel }} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ .Values.node.kubelet | default "/var/lib/kubelet" }}/plugins/efs.csi.aws.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + {{- with .Values.sidecars.nodeDriverRegistrar.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=/csi/csi.sock + - --health-port={{ .Values.node.healthPort }} + - --v={{ .Values.node.logLevel }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- with .Values.sidecars.livenessProbe.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.livenessProbe.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: {{ .Values.node.kubelet | default "/var/lib/kubelet" }} + type: Directory + - name: plugin-dir + hostPath: + path: {{ .Values.node.kubelet | default "/var/lib/kubelet" }}/plugins/efs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ .Values.node.kubelet | default "/var/lib/kubelet" }}/plugins_registry/ + type: Directory + - name: efs-state-dir + hostPath: + path: /var/run/efs + type: DirectoryOrCreate + - name: efs-utils-config + hostPath: + path: /var/amazon/efs + type: DirectoryOrCreate + - name: efs-utils-config-legacy + hostPath: + path: /etc/amazon/efs + type: DirectoryOrCreate +{{- end }} +{{- end }} diff --git a/charts/aws-efs-csi-driver/templates/node-daemonset.yaml b/charts/aws-efs-csi-driver/templates/node-daemonset.yaml index 5eb8a9bde..cc71906ba 100644 --- a/charts/aws-efs-csi-driver/templates/node-daemonset.yaml +++ b/charts/aws-efs-csi-driver/templates/node-daemonset.yaml @@ -1,184 +1,14 @@ -# Node Service -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: efs-csi-node - labels: - app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} -spec: - selector: - matchLabels: - app: efs-csi-node - app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - {{- with .Values.node.updateStrategy }} - updateStrategy: - {{ toYaml . | nindent 4 }} - {{- end }} - template: - metadata: - labels: - app: efs-csi-node - app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - {{- if .Values.node.podAnnotations }} - annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} - {{- end }} - spec: - {{- with .Values.node.hostAliases }} - hostAliases: - {{- range $k, $v := . }} - - ip: {{ $v.ip }} - hostnames: - - {{ $k }}.efs.{{ $v.region }}.amazonaws.com - {{- end }} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - nodeSelector: - kubernetes.io/os: linux - {{- with .Values.node.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.node.affinity }} - affinity: {{- toYaml . | nindent 8 }} - {{- end }} - hostNetwork: true - dnsPolicy: {{ .Values.node.dnsPolicy }} - {{- with .Values.node.dnsConfig }} - dnsConfig: {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ .Values.node.serviceAccount.name }} - priorityClassName: system-node-critical - {{- with .Values.node.tolerations }} - tolerations: {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.node.securityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: efs-plugin - securityContext: - privileged: true - image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v={{ .Values.node.logLevel }} - - --vol-metrics-opt-in={{ hasKey .Values.node "volMetricsOptIn" | ternary .Values.node.volMetricsOptIn false }} - - --vol-metrics-refresh-period={{ hasKey .Values.node "volMetricsRefreshPeriod" | ternary .Values.node.volMetricsRefreshPeriod 240 }} - - --vol-metrics-fs-rate-limit={{ hasKey .Values.node "volMetricsFsRateLimit" | ternary .Values.node.volMetricsFsRateLimit 5 }} - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock - - name: CSI_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- if .Values.useFIPS }} - - name: AWS_USE_FIPS_ENDPOINT - value: "true" - {{- end }} - volumeMounts: - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: plugin-dir - mountPath: /csi - - name: efs-state-dir - mountPath: /var/run/efs - - name: efs-utils-config - mountPath: /var/amazon/efs - - name: efs-utils-config-legacy - mountPath: /etc/amazon/efs-legacy - ports: - - name: healthz - containerPort: {{ .Values.node.healthPort }} - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 2 - failureThreshold: 5 - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - - name: csi-driver-registrar - image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} - imagePullPolicy: {{ .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.node.logLevel }} - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/efs.csi.aws.com/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - {{- with .Values.sidecars.nodeDriverRegistrar.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: liveness-probe - image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} - imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} - args: - - --csi-address=/csi/csi.sock - - --health-port={{ .Values.node.healthPort }} - - --v={{ .Values.node.logLevel }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - {{- with .Values.sidecars.livenessProbe.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.livenessProbe.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - volumes: - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/efs.csi.aws.com/ - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: efs-state-dir - hostPath: - path: /var/run/efs - type: DirectoryOrCreate - - name: efs-utils-config - hostPath: - path: /var/amazon/efs - type: DirectoryOrCreate - - name: efs-utils-config-legacy - hostPath: - path: /etc/amazon/efs - type: DirectoryOrCreate +{{$defaultArgs := dict + "NodeName" "efs-csi-node" +}} +{{- include "node" (deepCopy $ | mustMerge $defaultArgs) -}} +{{- range $name, $values := .Values.additionalDaemonSets }} +{{$args := dict + "NodeName" (printf "efs-csi-node-%s" $name) + "Values" (dict + "node" (deepCopy $.Values | mustMerge $values) + ) +}} +--- +{{- include "node" (deepCopy $ | mustMerge $args) -}} +{{- end }} diff --git a/charts/aws-efs-csi-driver/values.yaml b/charts/aws-efs-csi-driver/values.yaml index 18eba8e6a..17af3b326 100644 --- a/charts/aws-efs-csi-driver/values.yaml +++ b/charts/aws-efs-csi-driver/values.yaml @@ -163,6 +163,33 @@ node: runAsGroup: 0 fsGroup: 0 +additionalDaemonSets: +# Additional node DaemonSets, using the config structure +# See docs/additional-daemonsets.md for more information +# +# example: +# kubelet: /mnt/resource/kubelet +# tolerations: +# - operator: Exists +# nodeSelector: { } +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: eks.amazonaws.com/compute-type +# operator: NotIn +# values: +# - fargate +# - key: kubelet.kubernetes.io/directory-location +# operator: In +# values: +# - mnt-resource-kubelet +# - key: kubernetes.io/os +# operator: In +# values: +# - linux + storageClasses: [] # Add StorageClass resources like: # - name: efs-sc diff --git a/docs/additional-daemonsets.md b/docs/additional-daemonsets.md new file mode 100644 index 000000000..23f22f25c --- /dev/null +++ b/docs/additional-daemonsets.md @@ -0,0 +1,52 @@ +# Additional Node DaemonSets Feature + +In some situations, it is desirable to create multiple node `DaemonSet`s of the EFS CSI Driver. For example, when different AWS nodes in the cluster have different kubelet directory locations. + +The EFS CSI Driver Helm chart supports the creation of additional `DaemonSet`s via the `.additionalDaemonSets` parameter. Node configuration from the values supplied to `.node` are taken as a default, with the values supplied in the `.additionalDaemonSets` configuration as overrides. An additional `DaemonSet` will be rendered for each entry in `additionalDaemonSets`. + +**WARNING: The EFS CSI Driver does not support running multiple node pods on the same node. If you use this feature, ensure that all nodes are targeted by no more than one `DaemonSet`s.** + +## Example + +For example, the following configuration would produce two `DaemonSet`s: + +```yaml +node: + # Number for the log level verbosity + logLevel: 2 + volMetricsOptIn: false + volMetricsRefreshPeriod: 240 + volMetricsFsRateLimit: 5 + +additionalDaemonSets: + nodeGroup1: + kubelet: /mnt/resource/kubelet + tolerations: + - operator: Exists + nodeSelector: { } + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + - key: kubelet.kubernetes.io/directory-location + operator: In + values: + - mnt-resource-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux +``` + +The `DaemonSet`s would be configured as follows: + +- `efs-csi-node` (the default `DaemonSet`) +Will be configured to use default /var/lib/kubelet as the KubeletDirectory. +- `efs-csi-node-nodeGroup1` +Will be configured to use configured /mnt/resource/kubelet as the KubeletDirectory. +Note how the other config values is inherited from the `.node` configuration because this config does not specify them. This way, `.node` can be used to set defaults for all the `DaemonSet`s. \ No newline at end of file