From 259c4c2259e3c675fc6e33a2d8b0c4d721191868 Mon Sep 17 00:00:00 2001 From: lucianvlad <88718063+lucianvlad@users.noreply.github.com> Date: Mon, 25 Sep 2023 07:37:36 +0200 Subject: [PATCH] Patch 0.32.1 (#1783) Co-authored-by: Cristi <77267293+crssnd@users.noreply.github.com> Co-authored-by: linus-astrom <129269947+linus-astrom@users.noreply.github.com> Co-authored-by: raviranjan --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 15 +++ bin/update-ips.bash | 13 +- config/config/sc-config.yaml | 36 ++++-- config/secrets/sc-secrets.yaml | 4 +- .../templates/rclone-sync.yaml | 17 +-- .../templates/alerts/kubernetes-apps.yaml | 2 +- helmfile/charts/rclone-sync/files/rclone.conf | 14 +- helmfile/charts/rclone-sync/values.yaml | 4 + .../networkpolicy/service-cluster.yaml.gotmpl | 81 +++++++++--- helmfile/values/rclone-sync.yaml.gotmpl | 120 ++++++++++++++---- migration/v0.32/README.md | 40 ++++++ .../prepare/20-sync-type-new-key-name.sh | 10 ++ .../prepare/20-sync-type-new-key-name.sh | 10 ++ scripts/migration/lib.sh | 14 ++ 15 files changed, 302 insertions(+), 80 deletions(-) create mode 100755 migration/v0.32/prepare/20-sync-type-new-key-name.sh create mode 100755 migration/v0.33/prepare/20-sync-type-new-key-name.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ae957f98..7eb95f338 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: exclude: ^helmfile/upstream/|^CHANGELOG.md$|^WIP-CHANGELOG.md$|^helmfile/charts/grafana-ops/files/welcome.md$|^helmfile/charts/opensearch/configurer/files/dashboards-resources/welcome.md$|^images/elasticsearch-curator/README.md args: - -r - - ~MD013,~MD024,~MD026,~MD028,~MD034 + - ~MD013,~MD024,~MD026,~MD028,~MD034,~MD033 - repo: https://github.com/codespell-project/codespell rev: v2.2.4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 831150aa2..e0fa8ad17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Compliant Kubernetes changelog +- [v0.32.1](#v0321---2023-09-19) - [v0.32.0](#v0320---2023-08-07) - [v0.31.0](#v0310---2023-07-06) - [v0.30.0](#v0300---2023-05-10) @@ -36,6 +37,20 @@ - [v0.5.0](#v050---2020-08-06) +------------------------------------------------- +## v0.32.1 - 2023-09-19 + +### Release notes + +### Changed + +- If sync is enabled and swift is used for Harbor or Thanos then the sync job will automatically use swift +- Changed the alert `KubeContainerOOMKilled` threshold. +- Allow domains to be used for swift in rclone-sync + +### Fixed +- Only check for swift if sync default buckets is enabled with harbor or thanos using swift + ------------------------------------------------- ## v0.32.0 - 2023-08-07 diff --git a/bin/update-ips.bash b/bin/update-ips.bash index a951b2610..6890bc6c2 100755 --- a/bin/update-ips.bash +++ b/bin/update-ips.bash @@ -394,6 +394,8 @@ fi ## Add destination object storage ips for rclone sync to sc config if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then if [ "$(yq_dig 'sc' '.networkPolicies.rcloneSync.enabled' 'false')" == "true" ]; then + check_harbor="$(yq_dig 'sc' '.harbor.persistence.type' 'false')" + check_thanos="$(yq_dig 'sc' '.thanos.objectStorage.type' 'false')" destination=$(yq4 '.objectStorage.sync.buckets.[].destinationType' "${config["override_sc"]}") destinationSwift=false destinationS3=false @@ -404,6 +406,10 @@ if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then destinationS3=true fi done + if [ "$check_harbor" == "swift" ] || [ "$check_thanos" == "swift" ]; then + destinationSwift=true + fi + ifNull="" S3_ENDPOINT_DST="$(yq_dig 'sc' '.objectStorage.sync.s3.regionEndpoint' "" | sed 's/https\?:\/\///' | sed 's/[:\/].*//')" S3_PORT_DST="$(yq_dig 'sc' '.objectStorage.sync.s3.regionEndpoint' "" | sed 's/https\?:\/\///' | sed 's/[A-Za-z.0-9-]*:\?//' | sed 's/\/.*//')" @@ -411,7 +417,7 @@ if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then SWIFT_ENDPOINT_DST="$(yq_dig 'sc' '.objectStorage.sync.swift.authUrl' "" | sed 's/https\?:\/\///' | sed 's/[:\/].*//')" SWIFT_PORT_DST="$(yq_dig 'sc' '.objectStorage.sync.swift.authUrl' "" | sed 's/https\?:\/\///' | sed 's/[A-Za-z.0-9-]*:\?//' | sed 's/\/.*//')" - if { [ "$destinationS3" == "true" ] && [ "$destinationSwift" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.type' 's3')" == "s3" ]; }; then + if { [ "$destinationS3" == "true" ] && [ "$destinationSwift" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.destinationType' 's3')" == "s3" ]; }; then if [ -z "${S3_ENDPOINT_DST}" ]; then log_error "No destination S3 endpoint for rclone sync found, check your sc-config.yaml" exit 1 @@ -435,7 +441,7 @@ if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then fi ifNull=true fi - if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.type' 'swift')" == "swift" ]; }; then + if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.desinationType' 'swift')" == "swift" ]; }; then if [ -z "${SWIFT_ENDPOINT_DST}" ]; then log_error "No destination Swift endpoint for rclone sync found, check your sc-config.yaml" exit 1 @@ -460,9 +466,8 @@ if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then checkIfDiffAndUpdateDNSIPs "${SWIFT_ENDPOINT_DST}" ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ips" "${config["override_sc"]}" checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ports" "${config["override_sc"]}" "$SWIFT_PORT_DST" ifNull=true - fi - if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" == "true" ]; } || [ -z "$ifNull" ] && { [ "$(yq_dig 'sc' '.objectStorage.sync.type' 'swift')" == "swift" ] || [ "$(yq_dig 'sc' '.objectStorage.sync.type' 's3')" == "s3" ]; }; then + if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" == "true" ]; } || [ -z "$ifNull" ] && { [ "$(yq_dig 'sc' '.objectStorage.sync.destinationType' 'swift')" == "swift" ] || [ "$(yq_dig 'sc' '.objectStorage.sync.destinationType' 's3')" == "s3" ]; }; then if [ -z "${S3_ENDPOINT_DST}" ]; then log_error "No destination S3 endpoint for rclone sync found, check your sc-config.yaml" exit 1 diff --git a/config/config/sc-config.yaml b/config/config/sc-config.yaml index 70d157ff5..f7c278f40 100644 --- a/config/config/sc-config.yaml +++ b/config/config/sc-config.yaml @@ -32,8 +32,9 @@ objectStorage: enabled: false # dryrun: false - ## Options are 's3 and swift' - type: none + ## Only 's3' is currently supported for `.objectStorage.sync.destinationType` as not all default buckets applications have swift support. + ## If Harbor or Thanos are using swift then we will automatically use swift for the sync, regardless of the value set for type. + destinationType: s3 # secondaryUrl: set-me if regionEndpoint and or authUrl does not have all the relevant ips and or ports used for rclone-sync networkpolicy. # s3: # region: set-me @@ -45,6 +46,7 @@ objectStorage: # swift: # authUrl: set-me # region: set-me + # projectName: set-me ## Sync all buckets under 'objectStorage.buckets' ## These will be appended to 'buckets' using the same name from source as destination, and the default schedule. syncDefaultBuckets: false @@ -1146,17 +1148,19 @@ welcomingDashboard: networkPolicies: global: objectStorageSwift: - ips: [] # - set-me-if-enabled - ports: [] # - set-me-if-enabled + ips: + - "set-me-if-enabled" + ports: + - 5000 scApiserver: # usually private ip of control-plane nodes ips: - - set-me + - "set-me" port: 6443 scNodes: # ip of all nodes in the cluster for internal communication ips: - - set-me + - "set-me" harbor: enabled: true # For replication, added to core and jobservice @@ -1243,14 +1247,20 @@ networkPolicies: rcloneSync: enabled: true destinationObjectStorageS3: - ips: [] # - "set-me-if-objectStorage.sync.enabled-and-type-is-s3" - ports: [] # - 443 + ips: + - "set-me-if-objectStorage.sync.enabled-and-type-is-s3" + ports: + - 443 destinationObjectStorageSwift: - ips: [] # - "set-me-if-objectStorage.sync.enabled-and-type-is-swift" - ports: [] # - 443 + ips: + - "set-me-if-objectStorage.sync.enabled-type-is-swift-or-harbor-thanos-use-swift" + ports: + - 5000 secondaryUrl: - ips: [] # - "set-me-if-secondaryUrl-has-a-url" - ports: [] # - 443 + ips: + - "set-me-if-secondaryUrl-has-an-url" + ports: + - 443 s3Exporter: enabled: true @@ -1265,7 +1275,7 @@ networkPolicies: ingressNginx: ingressOverride: ips: - - set-me-if-enabled + - "set-me-if-enabled" dex: enabled: true # Ip to connector, e.g. Google, LDAP, ... diff --git a/config/secrets/sc-secrets.yaml b/config/secrets/sc-secrets.yaml index afd787255..d22e5ce3e 100644 --- a/config/secrets/sc-secrets.yaml +++ b/config/secrets/sc-secrets.yaml @@ -18,8 +18,10 @@ objectStorage: {} # accessKey: "set-me" # secretKey: "set-me" # swift: - # applicationCredentialID: "set-me" + # applicationCredentialID: "set-me" # application credentials are preferred over username and password, but both are currently supported # applicationCredentialSecret: "set-me" + # username: "set-me" + # password: "set-me" # encrypt: # password: "set-me" # generate with `pwgen 32 1` # salt: "set-me" # generate with `pwgen 32 1` diff --git a/helmfile/charts/networkpolicy/service-cluster/templates/rclone-sync.yaml b/helmfile/charts/networkpolicy/service-cluster/templates/rclone-sync.yaml index f563d14b3..0973734e2 100644 --- a/helmfile/charts/networkpolicy/service-cluster/templates/rclone-sync.yaml +++ b/helmfile/charts/networkpolicy/service-cluster/templates/rclone-sync.yaml @@ -16,25 +16,14 @@ spec: protocol: UDP - port: 53 protocol: TCP - {{- if and .Values.global.objectStorage.ips .Values.global.objectStorage.ports }} + {{- if and .Values.rcloneSync.objectStorage.ips .Values.rcloneSync.objectStorage.ports }} - to: - {{- range .Values.global.objectStorage.ips | uniq }} + {{- range .Values.rcloneSync.objectStorage.ips | uniq }} - ipBlock: cidr: {{ . }} {{- end }} ports: - {{- range .Values.global.objectStorage.ports | uniq }} - - port: {{ . }} - {{- end }} - {{- end }} - {{- if and .Values.rcloneSync.destinationObjectStorage.ips .Values.rcloneSync.destinationObjectStorage.ports }} - - to: - {{- range .Values.rcloneSync.destinationObjectStorage.ips | uniq }} - - ipBlock: - cidr: {{ . }} - {{- end }} - ports: - {{- range .Values.rcloneSync.destinationObjectStorage.ports | uniq }} + {{- range .Values.rcloneSync.objectStorage.ports | uniq }} - port: {{ . }} {{- end }} {{- end }} diff --git a/helmfile/charts/prometheus-alerts/templates/alerts/kubernetes-apps.yaml b/helmfile/charts/prometheus-alerts/templates/alerts/kubernetes-apps.yaml index 1cca7adfd..6752fd397 100644 --- a/helmfile/charts/prometheus-alerts/templates/alerts/kubernetes-apps.yaml +++ b/helmfile/charts/prometheus-alerts/templates/alerts/kubernetes-apps.yaml @@ -53,7 +53,7 @@ spec: annotations: description: Container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} from {{`{{`}} $labels.cluster {{`}}`}} has been OOMKilled {{`{{`}} $value {{`}}`}} times in the last 10 minutes. summary: Kubernetes container OOMKilled. - expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1 + expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 30m >= 2) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[30m]) == 1 for: 0m labels: severity: warning diff --git a/helmfile/charts/rclone-sync/files/rclone.conf b/helmfile/charts/rclone-sync/files/rclone.conf index 02a1e8f7f..762b7e1e1 100644 --- a/helmfile/charts/rclone-sync/files/rclone.conf +++ b/helmfile/charts/rclone-sync/files/rclone.conf @@ -12,10 +12,22 @@ v2_auth = {{ .s3.v2Auth | default false }} {{- end }} {{- if eq .type "swift" }} +{{- if and (get .swift "applicationCredentialID") (get .swift "applicationCredentialSecret") }} application_credential_id = {{ .swift.applicationCredentialID }} application_credential_secret = {{ .swift.applicationCredentialSecret }} +{{- else }} +user = {{.swift.username }} +key = {{.swift.password }} +{{- end }} auth = {{ .swift.authUrl }} region = {{ .swift.region }} +tenant = {{ .swift.tenant }} +{{- if get .swift "domainName" }} +domain = {{ .swift.domainName }} +{{- end }} +{{- if get .swift "projectDomainName" }} +tenant_domain = {{ .swift.projectDomainName }} +{{- end }} {{- end }} {{- end }} @@ -24,7 +36,7 @@ region = {{ .swift.region }} [{{ $.Values.config.encrypt.name }}-{{ .destination }}] type = crypt -remote = {{ $.Values.config.destination.name }}:{{ .destination }} +remote = dest-{{ .destinationType }}:{{ .destination }} password = {{ $.Values.config.encrypt.password }} password2 = {{ $.Values.config.encrypt.salt }} {{- if not $.Values.config.encrypt.directoryNames }} diff --git a/helmfile/charts/rclone-sync/values.yaml b/helmfile/charts/rclone-sync/values.yaml index 4e24c91c0..048de27c4 100644 --- a/helmfile/charts/rclone-sync/values.yaml +++ b/helmfile/charts/rclone-sync/values.yaml @@ -22,6 +22,8 @@ config: applicationCredentialSecret: secret-key authUrl: swift.region.provider.net region: region + # domainName: domain-name + # projectDomainName: project-domain-name destination: - name: provider-region-b @@ -40,6 +42,8 @@ config: applicationCredentialSecret: secret-key authUrl: swift.region.provider.net region: region + # domainName: domain-name + # projectDomainName: project-domain-name encrypt: enabled: false diff --git a/helmfile/values/networkpolicy/service-cluster.yaml.gotmpl b/helmfile/values/networkpolicy/service-cluster.yaml.gotmpl index d0e5b9a75..a206e1999 100644 --- a/helmfile/values/networkpolicy/service-cluster.yaml.gotmpl +++ b/helmfile/values/networkpolicy/service-cluster.yaml.gotmpl @@ -9,20 +9,14 @@ global: wcIngress: ips: {{- toYaml .Values.networkPolicies.global.wcIngress.ips | nindent 6 }} objectStorage: - {{ $sourceSwift := "" }} - {{- range .Values.objectStorage.sync.buckets }} - {{- if or (eq (get "sourceType" "" .) "swift") (and (ne (get "sourceType" "" .) "s3") (eq $.Values.objectStorage.type "swift") ) }} - {{ $sourceSwift = "true" }} - {{- end }} - {{- end }} ips: {{- toYaml .Values.networkPolicies.global.objectStorage.ips | nindent 6 }} - {{- if and (or (eq .Values.harbor.persistence.type "swift") (eq .Values.thanos.objectStorage.type "swift") ($sourceSwift)) (.Values.networkPolicies.global.objectStorageSwift.ips) }} + {{- if and (or (eq .Values.harbor.persistence.type "swift") (eq .Values.thanos.objectStorage.type "swift")) (.Values.networkPolicies.global.objectStorageSwift.ips) }} {{- toYaml .Values.networkPolicies.global.objectStorageSwift.ips | nindent 6 }} {{- end }} ports: {{- toYaml .Values.networkPolicies.global.objectStorage.ports | nindent 6 }} - {{- if and (or (eq .Values.harbor.persistence.type "swift") (eq .Values.thanos.objectStorage.type "swift") ($sourceSwift)) (.Values.networkPolicies.global.objectStorageSwift.ports) }} + {{- if and (or (eq .Values.harbor.persistence.type "swift") (eq .Values.thanos.objectStorage.type "swift")) (.Values.networkPolicies.global.objectStorageSwift.ports) }} {{- toYaml .Values.networkPolicies.global.objectStorageSwift.ports | nindent 6 }} {{- end }} externalLoadBalancer: {{ .Values.networkPolicies.global.externalLoadBalancer }} @@ -74,29 +68,82 @@ fluentd: rcloneSync: enabled: {{ and .Values.objectStorage.sync.enabled .Values.networkPolicies.rcloneSync.enabled }} - {{- if and .Values.objectStorage.sync.enabled .Values.networkPolicies.rcloneSync.enabled }} - destinationObjectStorage: + + {{ $sourceS3 := "" }} + {{ $sourceSwift := "" }} + {{ $destinationS3 := "" }} + {{ $destinationSwift := "" }} + + {{- if or (eq .Values.thanos.objectStorage.type "swift") (eq .Values.harbor.persistence.type "swift") }} + {{ $sourceSwift = "true" }} + {{ $destinationSwift = "true" }} + {{- end }} + + {{- if .Values.objectStorage.sync.syncDefaultBuckets }} + {{- if eq .Values.objectStorage.type "s3" }} + {{ $sourceS3 = "true" }} + {{- end }} + {{- if eq $.Values.objectStorage.sync.destinationType "s3" }} + {{ $destinationS3 := "true" }} + {{- else if eq $.Values.objectStorage.sync.destinationType "swift" }} + {{ $sourceSwift = "true" }} + {{- end }} + {{- end }} + + {{- if .Values.objectStorage.sync.buckets }} + {{- range .Values.objectStorage.sync.buckets }} + {{- if not (hasKey . "sourceType") }} + {{- $_ := set . "sourceType" $.Values.objectStorage.type }} + {{- end }} + {{- if eq .sourceType "s3" }} + {{ $sourceS3 = "true" }} + {{- else if eq .sourceType "swift" }} + {{ $sourceSwift = "true" }} + {{- end }} + + {{- if not (hasKey . "destinationType") }} + {{- $_ := set . "destinationType" $.Values.objectStorage.sync.destinationType }} + {{- end }} + {{- if eq .destinationType "s3" }} + {{ $destinationS3 = "true" }} + {{- else if eq .destinationType "swift" }} + {{ $destinationSwift = "true" }} + {{- end }} + {{- end }} + {{- end }} + objectStorage: ips: - {{- if and (hasKey .Values.objectStorage.sync.s3 "regionEndpoint" ) (.Values.objectStorage.sync.s3.regionEndpoint) (.Values.networkPolicies.rcloneSync.destinationObjectStorageS3.ips) }} + {{- if $sourceS3 }} + {{- toYaml .Values.networkPolicies.global.objectStorage.ips | nindent 6 }} + {{- end }} + {{- if $sourceSwift }} + {{- toYaml .Values.networkPolicies.global.objectStorageSwift.ips | nindent 6 }} + {{- end }} + {{- if $destinationS3 }} {{- toYaml .Values.networkPolicies.rcloneSync.destinationObjectStorageS3.ips | nindent 6 }} {{- end }} - {{- if and (hasKey .Values.objectStorage.sync.swift "authUrl" ) (.Values.objectStorage.sync.swift.authUrl) (.Values.networkPolicies.rcloneSync.destinationObjectStorageSwift.ips) }} + {{- if $destinationSwift }} {{- toYaml .Values.networkPolicies.rcloneSync.destinationObjectStorageSwift.ips | nindent 6 }} {{- end }} - {{- if and (hasKey .Values.objectStorage.sync "secondaryUrl" ) (.Values.objectStorage.sync.secondaryUrl) (.Values.networkPolicies.rcloneSync.secondaryUrl.ips) }} + {{- if hasKey .Values.objectStorage.sync "secondaryUrl" }} {{- toYaml .Values.networkPolicies.rcloneSync.secondaryUrl.ips | nindent 6 }} {{- end }} ports: - {{- if and (hasKey .Values.objectStorage.sync.s3 "regionEndpoint" ) (.Values.objectStorage.sync.s3.regionEndpoint) (.Values.networkPolicies.rcloneSync.destinationObjectStorageS3.ports) }} + {{- if $sourceS3 }} + {{- toYaml .Values.networkPolicies.global.objectStorage.ports | nindent 6 }} + {{- end }} + {{- if $sourceSwift }} + {{- toYaml .Values.networkPolicies.global.objectStorageSwift.ports | nindent 6 }} + {{- end }} + {{- if $destinationS3 }} {{- toYaml .Values.networkPolicies.rcloneSync.destinationObjectStorageS3.ports | nindent 6 }} {{- end }} - {{- if and (hasKey .Values.objectStorage.sync.swift "authUrl" ) (.Values.objectStorage.sync.swift.authUrl) (.Values.networkPolicies.rcloneSync.destinationObjectStorageSwift.ports) }} + {{- if $destinationSwift }} {{- toYaml .Values.networkPolicies.rcloneSync.destinationObjectStorageSwift.ports | nindent 6 }} {{- end }} - {{- if and (hasKey .Values.objectStorage.sync "secondaryUrl" ) (.Values.objectStorage.sync.secondaryUrl) (.Values.networkPolicies.rcloneSync.secondaryUrl.ports) }} + {{- if hasKey .Values.objectStorage.sync "secondaryUrl" }} {{- toYaml .Values.networkPolicies.rcloneSync.secondaryUrl.ports | nindent 6 }} {{- end }} - {{- end }} s3Exporter: enabled: {{ and (eq .Values.objectStorage.type "s3") (and .Values.s3Exporter.enabled .Values.networkPolicies.s3Exporter.enabled) }} diff --git a/helmfile/values/rclone-sync.yaml.gotmpl b/helmfile/values/rclone-sync.yaml.gotmpl index 9ff5d4c69..c8cb9f7f2 100644 --- a/helmfile/values/rclone-sync.yaml.gotmpl +++ b/helmfile/values/rclone-sync.yaml.gotmpl @@ -1,10 +1,6 @@ -{{- range .Values.objectStorage.sync.buckets }} -{{- if or (and (ne (get "sourceType" "s3" .) "s3") (ne (get "sourceType" "swift" .) "swift") ) (and (ne $.Values.objectStorage.type "s3") (ne $.Values.objectStorage.type "swift") ) }} -{{- fail "rclone-sync only supports using s3 and swift on source ends" }} -{{- end }} -{{- if or (and (ne (get "destinationType" "s3" .) "s3") (ne (get "destinationType" "swift" .) "swift") ) (and (ne $.Values.objectStorage.sync.type "s3") (ne $.Values.objectStorage.sync.type "swift") ) }} -{{- fail "rclone-sync only supports using s3 and swift on destination ends" }} -{{- end }} +{{- $valid := list "s3" "swift" }} +{{- if not (and (eq $.Values.objectStorage.type "s3") (has $.Values.objectStorage.sync.destinationType $valid)) }} +{{- fail "rclone-sync only supports using s3 and swift for source and destination" }} {{- end }} {{- if not (or .Values.objectStorage.sync.syncDefaultBuckets .Values.objectStorage.sync.buckets) }} @@ -20,19 +16,51 @@ config: {{ $sourceSwift := "" }} {{ $destinationS3 := "" }} {{ $destinationSwift := "" }} +{{ $swiftEnabled := list }} -{{- range .Values.objectStorage.sync.buckets }} -{{- if or (eq (get "sourceType" "" .) "s3") (and (ne (get "sourceType" "" .) "swift") (eq $.Values.objectStorage.type "s3") ) }} -{{ $sourceS3 = "true" }} -{{- else if or (eq (get "sourceType" "" .) "swift") (and (ne (get "sourceType" "" .) "s3") (eq $.Values.objectStorage.type "swift") ) }} -{{ $sourceSwift = "true" }} +{{- if .Values.objectStorage.sync.syncDefaultBuckets }} + {{- if eq .Values.thanos.objectStorage.type "swift" }} + {{ $swiftEnabled = append $swiftEnabled .Values.objectStorage.buckets.thanos }} + {{ $sourceSwift = "true" }} + {{ $destinationSwift = "true" }} + {{- end }} + {{- if eq .Values.harbor.persistence.type "swift" }} + {{ $swiftEnabled = append $swiftEnabled .Values.objectStorage.buckets.harbor }} + {{ $sourceSwift = "true" }} + {{ $destinationSwift = "true" }} + {{- end }} + {{- if eq .Values.objectStorage.type "s3" }} + {{ $sourceS3 = "true" }} + {{- end }} + {{- if eq $.Values.objectStorage.sync.destinationType "s3" }} + {{ $destinationS3 = "true" }} + {{- else if eq $.Values.objectStorage.sync.destinationType "swift" }} + {{ $destinationSwift = "true" }} + {{- end }} {{- end }} -{{- if or (eq (get "destinationType" "" .) "s3") (and (ne (get "destinationType" "" .) "swift") (eq $.Values.objectStorage.sync.type "s3") ) }} -{{ $destinationS3 = "true" }} -{{- else if or (eq (get "destinationType" "" .) "swift") (and (ne (get "destinationType" "" .) "s3") (eq $.Values.objectStorage.sync.type "swift") ) }} -{{ $destinationSwift = "true" }} -{{- end }} +{{- range .Values.objectStorage.sync.buckets }} + {{- if not (hasKey . "sourceType") }} + {{- $_ := set . "sourceType" $.Values.objectStorage.type }} + {{- end }} + {{- if eq .sourceType "s3" }} + {{ $sourceS3 = "true" }} + {{- else if eq .sourceType "swift" }} + {{ $sourceSwift = "true" }} + {{- else }} + {{- fail "rclone-sync custom buckets only support s3 and swift for sourceType" }} + {{- end }} + + {{- if not (hasKey . "destinationType") }} + {{- $_ := set . "destinationType" $.Values.objectStorage.sync.destinationType }} + {{- end }} + {{- if eq .destinationType "s3" }} + {{ $destinationS3 = "true" }} + {{- else if eq .destinationType "swift" }} + {{ $destinationSwift = "true" }} + {{- else }} + {{- fail "rclone-sync custom buckets only support s3 and swift for destinationType" }} + {{- end }} {{- end }} source: @@ -51,13 +79,26 @@ config: {{- end }} {{- if $sourceSwift }} + {{- $swift := .Values.objectStorage | getOrNil "swift" | required "Swift source enabled for rclone but not configured!" }} - name: src-swift type: swift swift: - applicationCredentialID: {{ .Values.objectStorage.swift.applicationCredentialID }} - applicationCredentialSecret: {{ .Values.objectStorage.swift.applicationCredentialSecret }} - authUrl: {{ .Values.objectStorage.swift.authUrl }} - region: {{ .Values.objectStorage.swift.region }} + {{- if getOrNil "applicationCredentialID" $swift }} + applicationCredentialID: {{ $swift.applicationCredentialID }} + applicationCredentialSecret: {{ $swift.applicationCredentialSecret }} + {{- else }} + username: {{ $swift.username | quote }} + password: {{ $swift.password | quote }} + {{- end }} + authUrl: {{ $swift.authUrl }} + region: {{ $swift.region }} + tenant: {{ $swift.projectName }} + {{- if getOrNil "domainName" $swift }} + domainName: {{ $swift.domainName }} + {{- end }} + {{- if getOrNil "projectDomainName" $swift }} + projectDomainName: {{ $swift.projectDomainName }} + {{- end }} {{- end }} destination: @@ -70,18 +111,32 @@ config: region: {{ .Values.objectStorage.sync.s3.region }} regionEndpoint: {{ .Values.objectStorage.sync.s3.regionEndpoint }} forcePathStyle: {{ .Values.objectStorage.sync.s3.forcePathStyle }} - {{- if hasKey .Values.objectStorage.sync.s3 "v2Auth" }} + {{- if hasKey .Values.objectStorage.sync.s3 "v2Auth" }} v2Auth: {{ .Values.objectStorage.sync.s3.v2Auth }} - {{- end }} + {{- end }} {{- end }} + {{- if $destinationSwift }} + {{- $swift := .Values.objectStorage.sync | getOrNil "swift" | required "Swift destination enabled for rclone but not configured!" }} - name: dest-swift type: swift swift: - applicationCredentialID: {{ .Values.objectStorage.sync.swift.applicationCredentialID }} - applicationCredentialSecret: {{ .Values.objectStorage.sync.swift.applicationCredentialSecret }} - authUrl: {{ .Values.objectStorage.sync.swift.authUrl }} - region: {{ .Values.objectStorage.sync.swift.region }} + {{- if getOrNil "applicationCredentialID" $swift }} + applicationCredentialID: {{ $swift.applicationCredentialID }} + applicationCredentialSecret: {{ $swift.applicationCredentialSecret }} + {{- else }} + username: {{ $swift.username | quote }} + password: {{ $swift.password | quote }} + {{- end }} + authUrl: {{ $swift.authUrl }} + region: {{ $swift.region }} + tenant: {{ $swift.projectName }} + {{- if getOrNil "domainName" $swift }} + domainName: {{ $swift.domainName }} + {{- end }} + {{- if getOrNil "projectDomainName" $swift }} + projectDomainName: {{ $swift.projectDomainName }} + {{- end }} {{- end }} encrypt: @@ -101,8 +156,17 @@ defaultSchedule: {{ .Values.objectStorage.sync.defaultSchedule }} buckets: {{- if .Values.objectStorage.sync.syncDefaultBuckets }} {{- range (values .Values.objectStorage.buckets | sortAlpha) }} + {{- if has . $swiftEnabled }} + - source: {{ . }} + destination: {{ . }} + sourceType: swift + destinationType: swift + {{- else }} - source: {{ . }} destination: {{ . }} + sourceType: {{ $.Values.objectStorage.type }} + destinationType: {{ $.Values.objectStorage.sync.destinationType }} + {{- end }} {{- end }} {{- end }} {{- if .Values.objectStorage.sync.buckets }} @@ -117,7 +181,7 @@ buckets: schedule: {{ .schedule | quote }} {{- end }} sourceType: {{ . | dig "sourceType" $.Values.objectStorage.type }} - destinationType: {{ . | dig "destinationType" $.Values.objectStorage.sync.type }} + destinationType: {{ . | dig "destinationType" $.Values.objectStorage.sync.destinationType }} {{- end }} {{- end }} diff --git a/migration/v0.32/README.md b/migration/v0.32/README.md index 1632c74d8..bf4d7ff0f 100644 --- a/migration/v0.32/README.md +++ b/migration/v0.32/README.md @@ -35,6 +35,40 @@ As with all scripts in this repository `CK8S_CONFIG_PATH` is expected to be set. ## Prerequisites +- If `.objectStorage.sync.enabled: true` and `.objectStorage.sync.syncDefaultBuckets: true` and `.thanos.objectStorage.type: swift` or `.harbor.persistence.type: swift` then the rclone jobs will automatically use swift for Thanos and/or Harbor source and destination buckets. You will need to create the application credentials for swift and add them into the `secrets.yaml`: + +
Create source application credentials + + ```bash + source ${CK8S_CONFIG_PATH}/.sh + source <(sops -d ${CK8S_CONFIG_PATH}/secret/.sh) + + openstack application credential create -swift + + sops ${CK8S_CONFIG_PATH}/secrets.yaml + + # set objectStorage.swift.applicationCredentialID in secrets.yaml + # set objectStorage.swift.applicationCredentialSecret in secrets.yaml + ``` + +
+ +
Create destination application credentials + + ```bash + source ${CK8S_CONFIG_PATH}/.sh + source <(sops -d ${CK8S_CONFIG_PATH}/secret/.sh) + + openstack application credential create -swift + + sops ${CK8S_CONFIG_PATH}/secrets.yaml + + # set objectStorage.sync.swift.applicationCredentialID in secrets.yaml + # set objectStorage.sync.swift.applicationCredentialSecret in secrets.yaml + ``` + +
+ - [ ] Notify the users (if any) before the upgrade starts; - [ ] Check if there are any pending changes to the environment; - [ ] Check the state of the environment, pods, nodes and backup jobs: @@ -103,6 +137,12 @@ As with all scripts in this repository `CK8S_CONFIG_PATH` is expected to be set. ./migration/v0.32/prepare/10-thanos-distrib-memory-limit.sh ``` +1. Rename the key `.objectStorage.sync.type` to `objectStorage.sync.destinationType` + + ```bash + ./migration/v0.32/prepare/20-sync-type-new-key-name.sh + ``` + 1. Update rclone sync networkPolicy name: ```bash diff --git a/migration/v0.32/prepare/20-sync-type-new-key-name.sh b/migration/v0.32/prepare/20-sync-type-new-key-name.sh new file mode 100755 index 000000000..1dc5dab36 --- /dev/null +++ b/migration/v0.32/prepare/20-sync-type-new-key-name.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +HERE="$(dirname "$(readlink -f "${0}")")" +ROOT="$(readlink -f "${HERE}/../../../")" + +# shellcheck source=scripts/migration/lib.sh +source "${ROOT}/scripts/migration/lib.sh" + +log_info " - move .sync.type to .sync.destinationType" +yq_move sc .objectStorage.sync.type .objectStorage.sync.destinationType diff --git a/migration/v0.33/prepare/20-sync-type-new-key-name.sh b/migration/v0.33/prepare/20-sync-type-new-key-name.sh new file mode 100755 index 000000000..1dc5dab36 --- /dev/null +++ b/migration/v0.33/prepare/20-sync-type-new-key-name.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +HERE="$(dirname "$(readlink -f "${0}")")" +ROOT="$(readlink -f "${HERE}/../../../")" + +# shellcheck source=scripts/migration/lib.sh +source "${ROOT}/scripts/migration/lib.sh" + +log_info " - move .sync.type to .sync.destinationType" +yq_move sc .objectStorage.sync.type .objectStorage.sync.destinationType diff --git a/scripts/migration/lib.sh b/scripts/migration/lib.sh index c71f34d98..9eebcba6d 100644 --- a/scripts/migration/lib.sh +++ b/scripts/migration/lib.sh @@ -129,6 +129,20 @@ config_validate() { pass="false" fi done + + sync_enabled=$(yq4 '.objectStorage.sync.enabled' <<< "${CONFIG["${1}"]}") + sync_default_enabled=$(yq4 '.objectStorage.sync.syncDefaultBuckets' <<< "${CONFIG["${1}"]}") + if [[ "${1}" = "sc" ]] && [[ "${sync_enabled}" = "true" ]] && [[ "${sync_default_enabled}" = "true" ]]; then + log_info "checking sync swift" + + check_harbor="$(yq4 '.harbor.persistence.type' <<< "${CONFIG["${1}"]}")" + check_thanos="$(yq4 '.thanos.objectStorage.type' <<< "${CONFIG["${1}"]}")" + check_sync_swift="$(yq4 '.objectStorage.sync.swift' <<< "${CONFIG["${1}"]}")" + + if { [[ "${check_harbor}" = "swift" ]] || [[ "${check_thanos}" = "swift" ]]; } && [[ "${check_sync_swift}" = "null" ]]; then + log_error "error: swift is enabled for Harbor/Thanos, but .objectStorage.sync is missing swift configuration" + fi + fi ;; *)