Patch 0.32.1 (#1783)

Co-authored-by: Cristi <[email protected]> Co-authored-by: linus-astrom <[email protected]> Co-authored-by: raviranjan <[email protected]>
elastisys · Sep 25, 2023 · 259c4c2 · 259c4c2
1 parent 82653df
commit 259c4c2
Show file tree

Hide file tree

Showing 15 changed files with 302 additions and 80 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -32,7 +32,7 @@ repos:
  exclude: ^helmfile/upstream/|^CHANGELOG.md$|^WIP-CHANGELOG.md$|^helmfile/charts/grafana-ops/files/welcome.md$|^helmfile/charts/opensearch/configurer/files/dashboards-resources/welcome.md$|^images/elasticsearch-curator/README.md
  args:
  - -r
- - ~MD013,~MD024,~MD026,~MD028,~MD034
+ - ~MD013,~MD024,~MD026,~MD028,~MD034,~MD033
 
  - repo: https://github.com/codespell-project/codespell
  rev: v2.2.4

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,6 @@
 # Compliant Kubernetes changelog
 <!-- BEGIN TOC -->
+- [v0.32.1](#v0321---2023-09-19)
 - [v0.32.0](#v0320---2023-08-07)
 - [v0.31.0](#v0310---2023-07-06)
 - [v0.30.0](#v0300---2023-05-10)
@@ -36,6 +37,20 @@
 - [v0.5.0](#v050---2020-08-06)
 <!-- END TOC -->
 
+-------------------------------------------------
+## v0.32.1 - 2023-09-19
+
+### Release notes
+
+### Changed
+
+- If sync is enabled and swift is used for Harbor or Thanos then the sync job will automatically use swift
+- Changed the alert `KubeContainerOOMKilled` threshold.
+- Allow domains to be used for swift in rclone-sync
+
+### Fixed
+- Only check for swift if sync default buckets is enabled with harbor or thanos using swift
+
 -------------------------------------------------
 ## v0.32.0 - 2023-08-07
 

diff --git a/bin/update-ips.bash b/bin/update-ips.bash
@@ -394,6 +394,8 @@ fi
 ## Add destination object storage ips for rclone sync to sc config
 if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then
  if [ "$(yq_dig 'sc' '.networkPolicies.rcloneSync.enabled' 'false')" == "true" ]; then
+ check_harbor="$(yq_dig 'sc' '.harbor.persistence.type' 'false')"
+ check_thanos="$(yq_dig 'sc' '.thanos.objectStorage.type' 'false')"
  destination=$(yq4 '.objectStorage.sync.buckets.[].destinationType' "${config["override_sc"]}")
  destinationSwift=false
  destinationS3=false
@@ -404,14 +406,18 @@ if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then
  destinationS3=true
  fi
  done
+ if [ "$check_harbor" == "swift" ] || [ "$check_thanos" == "swift" ]; then
+ destinationSwift=true
+ fi
+
  ifNull=""
  S3_ENDPOINT_DST="$(yq_dig 'sc' '.objectStorage.sync.s3.regionEndpoint' "" | sed 's/https\?:\/\///' | sed 's/[:\/].*//')"
  S3_PORT_DST="$(yq_dig 'sc' '.objectStorage.sync.s3.regionEndpoint' "" | sed 's/https\?:\/\///' | sed 's/[A-Za-z.0-9-]*:\?//' | sed 's/\/.*//')"
 
  SWIFT_ENDPOINT_DST="$(yq_dig 'sc' '.objectStorage.sync.swift.authUrl' "" | sed 's/https\?:\/\///' | sed 's/[:\/].*//')"
  SWIFT_PORT_DST="$(yq_dig 'sc' '.objectStorage.sync.swift.authUrl' "" | sed 's/https\?:\/\///' | sed 's/[A-Za-z.0-9-]*:\?//' | sed 's/\/.*//')"
 
- if { [ "$destinationS3" == "true" ] && [ "$destinationSwift" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.type' 's3')" == "s3" ]; }; then
+ if { [ "$destinationS3" == "true" ] && [ "$destinationSwift" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.destinationType' 's3')" == "s3" ]; }; then
  if [ -z "${S3_ENDPOINT_DST}" ]; then
  log_error "No destination S3 endpoint for rclone sync found, check your sc-config.yaml"
  exit 1
@@ -435,7 +441,7 @@ if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then
  fi
  ifNull=true
  fi
- if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.type' 'swift')" == "swift" ]; }; then
+ if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.desinationType' 'swift')" == "swift" ]; }; then
  if [ -z "${SWIFT_ENDPOINT_DST}" ]; then
  log_error "No destination Swift endpoint for rclone sync found, check your sc-config.yaml"
  exit 1
@@ -460,9 +466,8 @@ if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then
  checkIfDiffAndUpdateDNSIPs "${SWIFT_ENDPOINT_DST}" ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ips" "${config["override_sc"]}"
  checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ports" "${config["override_sc"]}" "$SWIFT_PORT_DST"
  ifNull=true
-
  fi
- if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" == "true" ]; } || [ -z "$ifNull" ] && { [ "$(yq_dig 'sc' '.objectStorage.sync.type' 'swift')" == "swift" ] || [ "$(yq_dig 'sc' '.objectStorage.sync.type' 's3')" == "s3" ]; }; then
+ if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" == "true" ]; } || [ -z "$ifNull" ] && { [ "$(yq_dig 'sc' '.objectStorage.sync.destinationType' 'swift')" == "swift" ] || [ "$(yq_dig 'sc' '.objectStorage.sync.destinationType' 's3')" == "s3" ]; }; then
  if [ -z "${S3_ENDPOINT_DST}" ]; then
  log_error "No destination S3 endpoint for rclone sync found, check your sc-config.yaml"
  exit 1

diff --git a/config/config/sc-config.yaml b/config/config/sc-config.yaml
@@ -32,8 +32,9 @@ objectStorage:
  enabled: false
  # dryrun: false
 
- ## Options are 's3 and swift'
- type: none
+ ## Only 's3' is currently supported for `.objectStorage.sync.destinationType` as not all default buckets applications have swift support.
+ ## If Harbor or Thanos are using swift then we will automatically use swift for the sync, regardless of the value set for type.
+ destinationType: s3
  # secondaryUrl: set-me if regionEndpoint and or authUrl does not have all the relevant ips and or ports used for rclone-sync networkpolicy.
  # s3:
  # region: set-me
@@ -45,6 +46,7 @@ objectStorage:
  # swift:
  # authUrl: set-me
  # region: set-me
+ # projectName: set-me
  ## Sync all buckets under 'objectStorage.buckets'
  ## These will be appended to 'buckets' using the same name from source as destination, and the default schedule.
  syncDefaultBuckets: false
@@ -1146,17 +1148,19 @@ welcomingDashboard:
 networkPolicies:
  global:
  objectStorageSwift:
- ips: [] # - set-me-if-enabled
- ports: [] # - set-me-if-enabled
+ ips:
+ - "set-me-if-enabled"
+ ports:
+ - 5000
  scApiserver:
  # usually private ip of control-plane nodes
  ips:
- - set-me
+ - "set-me"
  port: 6443
  scNodes:
  # ip of all nodes in the cluster for internal communication
  ips:
- - set-me
+ - "set-me"
  harbor:
  enabled: true
  # For replication, added to core and jobservice
@@ -1243,14 +1247,20 @@ networkPolicies:
  rcloneSync:
  enabled: true
  destinationObjectStorageS3:
- ips: [] # - "set-me-if-objectStorage.sync.enabled-and-type-is-s3"
- ports: [] # - 443
+ ips:
+ - "set-me-if-objectStorage.sync.enabled-and-type-is-s3"
+ ports:
+ - 443
  destinationObjectStorageSwift:
- ips: [] # - "set-me-if-objectStorage.sync.enabled-and-type-is-swift"
- ports: [] # - 443
+ ips:
+ - "set-me-if-objectStorage.sync.enabled-type-is-swift-or-harbor-thanos-use-swift"
+ ports:
+ - 5000
  secondaryUrl:
- ips: [] # - "set-me-if-secondaryUrl-has-a-url"
- ports: [] # - 443
+ ips:
+ - "set-me-if-secondaryUrl-has-an-url"
+ ports:
+ - 443
  s3Exporter:
  enabled: true
 
@@ -1265,7 +1275,7 @@ networkPolicies:
  ingressNginx:
  ingressOverride:
  ips:
- - set-me-if-enabled
+  - "set-me-if-enabled"
  dex:
  enabled: true
  # Ip to connector, e.g. Google, LDAP, ...

diff --git a/config/secrets/sc-secrets.yaml b/config/secrets/sc-secrets.yaml
@@ -18,8 +18,10 @@ objectStorage: {}
  # accessKey: "set-me"
  # secretKey: "set-me"
  # swift:
- # applicationCredentialID: "set-me"
+ # applicationCredentialID: "set-me" # application credentials are preferred over username and password, but both are currently supported
  # applicationCredentialSecret: "set-me"
+ # username: "set-me"
+ # password: "set-me"
  # encrypt:
  # password: "set-me" # generate with `pwgen 32 1`
  # salt: "set-me" # generate with `pwgen 32 1`

diff --git a/helmfile/charts/networkpolicy/service-cluster/templates/rclone-sync.yaml b/helmfile/charts/networkpolicy/service-cluster/templates/rclone-sync.yaml
@@ -16,25 +16,14 @@ spec:
  protocol: UDP
  - port: 53
  protocol: TCP
- {{- if and .Values.global.objectStorage.ips .Values.global.objectStorage.ports }}
+ {{- if and .Values.rcloneSync.objectStorage.ips .Values.rcloneSync.objectStorage.ports }}
  - to:
- {{- range .Values.global.objectStorage.ips | uniq }}
+ {{- range .Values.rcloneSync.objectStorage.ips | uniq }}
  - ipBlock:
  cidr: {{ . }}
  {{- end }}
  ports:
- {{- range .Values.global.objectStorage.ports | uniq }}
- - port: {{ . }}
- {{- end }}
- {{- end }}
- {{- if and .Values.rcloneSync.destinationObjectStorage.ips .Values.rcloneSync.destinationObjectStorage.ports }}
- - to:
- {{- range .Values.rcloneSync.destinationObjectStorage.ips | uniq }}
- - ipBlock:
- cidr: {{ . }}
- {{- end }}
- ports:
- {{- range .Values.rcloneSync.destinationObjectStorage.ports | uniq }}
+ {{- range .Values.rcloneSync.objectStorage.ports | uniq }}
  - port: {{ . }}
  {{- end }}
  {{- end }}

diff --git a/helmfile/charts/prometheus-alerts/templates/alerts/kubernetes-apps.yaml b/helmfile/charts/prometheus-alerts/templates/alerts/kubernetes-apps.yaml
@@ -53,7 +53,7 @@ spec:
  annotations:
  description: Container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} from {{`{{`}} $labels.cluster {{`}}`}} has been OOMKilled {{`{{`}} $value {{`}}`}} times in the last 10 minutes.
  summary: Kubernetes container OOMKilled.
- expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1
+ expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 30m >= 2) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[30m]) == 1
  for: 0m
  labels:
  severity: warning

diff --git a/helmfile/charts/rclone-sync/files/rclone.conf b/helmfile/charts/rclone-sync/files/rclone.conf
@@ -12,10 +12,22 @@ v2_auth = {{ .s3.v2Auth | default false }}
 {{- end }}
 
 {{- if eq .type "swift" }}
+{{- if and (get .swift "applicationCredentialID") (get .swift "applicationCredentialSecret") }}
 application_credential_id = {{ .swift.applicationCredentialID }}
 application_credential_secret = {{ .swift.applicationCredentialSecret }}
+{{- else }}
+user = {{.swift.username }}
+key = {{.swift.password }}
+{{- end }}
 auth = {{ .swift.authUrl }}
 region = {{ .swift.region }}
+tenant = {{ .swift.tenant }}
+{{- if get .swift "domainName" }}
+domain = {{ .swift.domainName }}
+{{- end }}
+{{- if get .swift "projectDomainName" }}
+tenant_domain = {{ .swift.projectDomainName }}
+{{- end }}
 {{- end }}
 {{- end }}
 
@@ -24,7 +36,7 @@ region = {{ .swift.region }}
 
 [{{ $.Values.config.encrypt.name }}-{{ .destination }}]
 type = crypt
-remote = {{ $.Values.config.destination.name }}:{{ .destination }}
+remote = dest-{{ .destinationType }}:{{ .destination }}
 password = {{ $.Values.config.encrypt.password }}
 password2 = {{ $.Values.config.encrypt.salt }}
 {{- if not $.Values.config.encrypt.directoryNames }}

diff --git a/helmfile/charts/rclone-sync/values.yaml b/helmfile/charts/rclone-sync/values.yaml
@@ -22,6 +22,8 @@ config:
  applicationCredentialSecret: secret-key
  authUrl: swift.region.provider.net
  region: region
+ # domainName: domain-name
+ # projectDomainName: project-domain-name
 
  destination:
  - name: provider-region-b
@@ -40,6 +42,8 @@ config:
  applicationCredentialSecret: secret-key
  authUrl: swift.region.provider.net
  region: region
+ # domainName: domain-name
+ # projectDomainName: project-domain-name
 
  encrypt:
  enabled: false

diff --git a/helmfile/values/networkpolicy/service-cluster.yaml.gotmpl b/helmfile/values/networkpolicy/service-cluster.yaml.gotmpl
@@ -9,20 +9,14 @@ global:
  wcIngress:
  ips: {{- toYaml .Values.networkPolicies.global.wcIngress.ips | nindent 6 }}
  objectStorage:
- {{ $sourceSwift := "" }}
- {{- range .Values.objectStorage.sync.buckets }}
- {{- if or (eq (get "sourceType" "" .) "swift") (and (ne (get "sourceType" "" .) "s3") (eq $.Values.objectStorage.type "swift") ) }}
- {{ $sourceSwift = "true" }}
- {{- end }}
- {{- end }}
  ips:
  {{- toYaml .Values.networkPolicies.global.objectStorage.ips | nindent 6 }}
- {{- if and (or (eq .Values.harbor.persistence.type "swift") (eq .Values.thanos.objectStorage.type "swift") ($sourceSwift)) (.Values.networkPolicies.global.objectStorageSwift.ips) }}
+ {{- if and (or (eq .Values.harbor.persistence.type "swift") (eq .Values.thanos.objectStorage.type "swift")) (.Values.networkPolicies.global.objectStorageSwift.ips) }}
  {{- toYaml .Values.networkPolicies.global.objectStorageSwift.ips | nindent 6 }}
  {{- end }}
  ports:
  {{- toYaml .Values.networkPolicies.global.objectStorage.ports | nindent 6 }}
- {{- if and (or (eq .Values.harbor.persistence.type "swift") (eq .Values.thanos.objectStorage.type "swift") ($sourceSwift)) (.Values.networkPolicies.global.objectStorageSwift.ports) }}
+ {{- if and (or (eq .Values.harbor.persistence.type "swift") (eq .Values.thanos.objectStorage.type "swift")) (.Values.networkPolicies.global.objectStorageSwift.ports) }}
  {{- toYaml .Values.networkPolicies.global.objectStorageSwift.ports | nindent 6 }}
  {{- end }}
  externalLoadBalancer: {{ .Values.networkPolicies.global.externalLoadBalancer }}
@@ -74,29 +68,82 @@ fluentd:
 
 rcloneSync:
  enabled: {{ and .Values.objectStorage.sync.enabled .Values.networkPolicies.rcloneSync.enabled }}
- {{- if and .Values.objectStorage.sync.enabled .Values.networkPolicies.rcloneSync.enabled }}
- destinationObjectStorage:
+
+ {{ $sourceS3 := "" }}
+ {{ $sourceSwift := "" }}
+ {{ $destinationS3 := "" }}
+ {{ $destinationSwift := "" }}
+
+ {{- if or (eq .Values.thanos.objectStorage.type "swift") (eq .Values.harbor.persistence.type "swift") }}
+ {{ $sourceSwift = "true" }}
+ {{ $destinationSwift = "true" }}
+ {{- end }}
+
+ {{- if .Values.objectStorage.sync.syncDefaultBuckets }}
+ {{- if eq .Values.objectStorage.type "s3" }}
+ {{ $sourceS3 = "true" }}
+ {{- end }}
+ {{- if eq $.Values.objectStorage.sync.destinationType "s3" }}
+ {{ $destinationS3 := "true" }}
+ {{- else if eq $.Values.objectStorage.sync.destinationType "swift" }}
+ {{ $sourceSwift = "true" }}
+ {{- end }}
+ {{- end }}
+
+ {{- if .Values.objectStorage.sync.buckets }}
+ {{- range .Values.objectStorage.sync.buckets }}
+ {{- if not (hasKey . "sourceType") }}
+ {{- $_ := set . "sourceType" $.Values.objectStorage.type }}
+ {{- end }}
+ {{- if eq .sourceType "s3" }}
+ {{ $sourceS3 = "true" }}
+ {{- else if eq .sourceType "swift" }}
+ {{ $sourceSwift = "true" }}
+ {{- end }}
+
+ {{- if not (hasKey . "destinationType") }}
+ {{- $_ := set . "destinationType" $.Values.objectStorage.sync.destinationType }}
+ {{- end }}
+ {{- if eq .destinationType "s3" }}
+ {{ $destinationS3 = "true" }}
+ {{- else if eq .destinationType "swift" }}
+ {{ $destinationSwift = "true" }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ objectStorage:
  ips:
- {{- if and (hasKey .Values.objectStorage.sync.s3 "regionEndpoint" ) (.Values.objectStorage.sync.s3.regionEndpoint) (.Values.networkPolicies.rcloneSync.destinationObjectStorageS3.ips) }}
+ {{- if $sourceS3 }}
+ {{- toYaml .Values.networkPolicies.global.objectStorage.ips | nindent 6 }}
+ {{- end }}
+ {{- if $sourceSwift }}
+ {{- toYaml .Values.networkPolicies.global.objectStorageSwift.ips | nindent 6 }}
+ {{- end }}
+ {{- if $destinationS3 }}
  {{- toYaml .Values.networkPolicies.rcloneSync.destinationObjectStorageS3.ips | nindent 6 }}
  {{- end }}
- {{- if and (hasKey .Values.objectStorage.sync.swift "authUrl" ) (.Values.objectStorage.sync.swift.authUrl) (.Values.networkPolicies.rcloneSync.destinationObjectStorageSwift.ips) }}
+ {{- if $destinationSwift }}
  {{- toYaml .Values.networkPolicies.rcloneSync.destinationObjectStorageSwift.ips | nindent 6 }}
  {{- end }}
- {{- if and (hasKey .Values.objectStorage.sync "secondaryUrl" ) (.Values.objectStorage.sync.secondaryUrl) (.Values.networkPolicies.rcloneSync.secondaryUrl.ips) }}
+ {{- if hasKey .Values.objectStorage.sync "secondaryUrl" }}
  {{- toYaml .Values.networkPolicies.rcloneSync.secondaryUrl.ips | nindent 6 }}
  {{- end }}
  ports:
- {{- if and (hasKey .Values.objectStorage.sync.s3 "regionEndpoint" ) (.Values.objectStorage.sync.s3.regionEndpoint) (.Values.networkPolicies.rcloneSync.destinationObjectStorageS3.ports) }}
+ {{- if $sourceS3 }}
+ {{- toYaml .Values.networkPolicies.global.objectStorage.ports | nindent 6 }}
+ {{- end }}
+ {{- if $sourceSwift }}
+ {{- toYaml .Values.networkPolicies.global.objectStorageSwift.ports | nindent 6 }}
+ {{- end }}
+ {{- if $destinationS3 }}
  {{- toYaml .Values.networkPolicies.rcloneSync.destinationObjectStorageS3.ports | nindent 6 }}
  {{- end }}
- {{- if and (hasKey .Values.objectStorage.sync.swift "authUrl" ) (.Values.objectStorage.sync.swift.authUrl) (.Values.networkPolicies.rcloneSync.destinationObjectStorageSwift.ports) }}
+ {{- if $destinationSwift }}
  {{- toYaml .Values.networkPolicies.rcloneSync.destinationObjectStorageSwift.ports | nindent 6 }}
  {{- end }}
- {{- if and (hasKey .Values.objectStorage.sync "secondaryUrl" ) (.Values.objectStorage.sync.secondaryUrl) (.Values.networkPolicies.rcloneSync.secondaryUrl.ports) }}
+ {{- if hasKey .Values.objectStorage.sync "secondaryUrl" }}
  {{- toYaml .Values.networkPolicies.rcloneSync.secondaryUrl.ports | nindent 6 }}
  {{- end }}
- {{- end }}
 
 s3Exporter:
  enabled: {{ and (eq .Values.objectStorage.type "s3") (and .Values.s3Exporter.enabled .Values.networkPolicies.s3Exporter.enabled) }}