apps sc: Added support for swift in rclone-sync.

WIP-CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Added
 
+- Added support for Swift in rclone-sync.
+ - It is now possible to specify type (S3/Swift) per bucket.
+
 ### Changed
 
 - Increased window for `FrequentPacketsDroppedFromWorkload` and `FrequentPacketsDroppedToWorkload` alerts

bin/update-ips.bash

@@ -289,7 +289,6 @@ if [ "${CHECK_CLUSTER}" == "both" ]; then
 else
  DIG_CLUSTER="wc"
 fi
-
 S3_ENDPOINT="$(yq_dig "${DIG_CLUSTER}" '.objectStorage.s3.regionEndpoint' '""' | sed 's/https\?:\/\///' | sed 's/[:\/].*//')"
 if [[ "${S3_ENDPOINT}" == "" ]]; then
  log_error "No S3 endpoint found, check your common-config.yaml (or ${DIG_CLUSTER}-config.yaml)"
@@ -347,8 +346,14 @@ fi
 if [[ "${CHECK_CLUSTER}" =~ ^(sc|both)$ ]]; then
  check_harbor="$(yq_dig 'sc' '.harbor.persistence.type' 'false')"
  check_thanos="$(yq_dig 'sc' '.thanos.objectStorage.type' 'false')"
-
- if [ "$check_harbor" == "swift" ] || [ "$check_thanos" == "swift" ]; then
+ sourceType=$(yq4 '.objectStorage.sync.buckets.[].sourceType' "${config["override_sc"]}")
+ sourceSwift=false
+ for type in $sourceType; do
+ if [ "$type" == "swift" ]; then
+ sourceSwift=true
+ fi
+ done
+ if [ "$check_harbor" == "swift" ] || [ "$check_thanos" == "swift" ] || [ "${sourceSwift}" == "true" ]; then
  os_auth_endpoint="$(yq_dig 'sc' '.objectStorage.swift.authUrl' '""' | sed 's/https\?:\/\///' | sed 's/[:\/].*//')"
 
  if [ -z "$os_auth_endpoint" ]; then
@@ -389,20 +394,115 @@ fi
 ## Add destination object storage ips for rclone sync to sc config
 if [ "$(yq_dig 'sc' '.objectStorage.sync.enabled' 'false')" == "true" ]; then
  if [ "$(yq_dig 'sc' '.networkPolicies.rcloneSync.enabled' 'false')" == "true" ]; then
+ destination=$(yq4 '.objectStorage.sync.buckets.[].destinationType' "${config["override_sc"]}")
+ destinationSwift=false
+ destinationS3=false
+ for type in $destination; do
+ if [ "$type" == "swift" ]; then
+ destinationSwift=true
+ elif [ "$type" == "s3" ]; then
+ destinationS3=true
+ fi
+ done
+ ifNull=""
+ S3_ENDPOINT_DST="$(yq_dig 'sc' '.objectStorage.sync.s3.regionEndpoint' "" | sed 's/https\?:\/\///' | sed 's/[:\/].*//')"
+ S3_PORT_DST="$(yq_dig 'sc' '.objectStorage.sync.s3.regionEndpoint' "" | sed 's/https\?:\/\///' | sed 's/[A-Za-z.0-9-]*:\?//' | sed 's/\/.*//')"
+
+ SWIFT_ENDPOINT_DST="$(yq_dig 'sc' '.objectStorage.sync.swift.authUrl' "" | sed 's/https\?:\/\///' | sed 's/[:\/].*//')"
+ SWIFT_PORT_DST="$(yq_dig 'sc' '.objectStorage.sync.swift.authUrl' "" | sed 's/https\?:\/\///' | sed 's/[A-Za-z.0-9-]*:\?//' | sed 's/\/.*//')"
+
+ if { [ "$destinationS3" == "true" ] && [ "$destinationSwift" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.type' 's3')" == "s3" ]; }; then
+ if [ -z "${S3_ENDPOINT_DST}" ]; then
+ log_error "No destination S3 endpoint for rclone sync found, check your sc-config.yaml"
+ exit 1
+ fi
+ if [ -z "${S3_PORT_DST}" ]; then
+ S3_PORT_DST="443"
+ fi
+ checkIfDiffAndUpdateDNSIPs "${S3_ENDPOINT_DST}" ".networkPolicies.rcloneSync.destinationObjectStorageS3.ips" "${config["override_sc"]}"
+ checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.destinationObjectStorageS3.ports" "${config["override_sc"]}" "$S3_PORT_DST"
+ if [ -z "${SWIFT_ENDPOINT_DST}" ] && [ -z "${SWIFT_PORT_DST}" ] && [ $DRY_RUN == "true" ]; then
+ results_diff=$(diff -U0 --color=always <(yq4 -P 'sort_keys(.networkPolicies.rcloneSync.destinationObjectStorageSwift)' "${config["override_sc"]}") <(yq4 -P 'del(.networkPolicies.rcloneSync.destinationObjectStorageSwift)' "${config["override_sc"]}") --label "${config["override_sc"]//${CK8S_CONFIG_PATH}\//}" --label expected) || true
+ if [ "${results_diff}" != "" ]; then
+ printf "${results_diff}"'%s\n'
+ log_warning "Diff found for .networkPolicies.rcloneSync.destinationObjectStorageSwift in ${config[override_sc]//${CK8S_CONFIG_PATH}\//} (diff shows actions needed to be up to date)"
+ fi
+ elif [ -z "${SWIFT_ENDPOINT_DST}" ] && [ -z "${SWIFT_PORT_DST}" ] && [ $DRY_RUN != "true" ]; then
+ yq4 -i 'del(.networkPolicies.rcloneSync.destinationObjectStorageSwift)' "${config["override_sc"]}"
+ else
+ checkIfDiffAndUpdateDNSIPs "${SWIFT_ENDPOINT_DST}" ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ips" "${config["override_sc"]}"
+ checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ports" "${config["override_sc"]}" "$SWIFT_PORT_DST"
+ fi
+ ifNull=true
+ fi
+ if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" != "true" ]; } || { [ "$destinationS3" != "true" ] && [ "$destinationSwift" != "true" ] && [ "$(yq_dig 'sc' '.objectStorage.sync.type' 'swift')" == "swift" ]; }; then
+ if [ -z "${SWIFT_ENDPOINT_DST}" ]; then
+ log_error "No destination Swift endpoint for rclone sync found, check your sc-config.yaml"
+ exit 1
+ fi
+ if [ -z "${SWIFT_PORT_DST}" ]; then
+ SWIFT_PORT_DST="443"
+ fi
+
+ if [ -z "${S3_ENDPOINT_DST}" ] && [ -z "${S3_PORT_DST}" ] && [ $DRY_RUN == "true" ]; then
+ results_diff=$(diff -U0 --color=always <(yq4 -P 'sort_keys(.networkPolicies.rcloneSync.destinationObjectStorageS3)' "${config["override_sc"]}") <(yq4 -P 'del(.networkPolicies.rcloneSync.destinationObjectStorageS3)' "${config["override_sc"]}") --label "${config["override_sc"]//${CK8S_CONFIG_PATH}\//}" --label expected) || true
+ if [ "${results_diff}" != "" ]; then
+ printf "${results_diff}"'%s\n'
+ log_warning "Diff found for .networkPolicies.rcloneSync.destinationObjectStorageS3 in ${config[override_sc]//${CK8S_CONFIG_PATH}\//} (diff shows actions needed to be up to date)"
+ fi
+ elif [ -z "${S3_ENDPOINT_DST}" ] && [ -z "${S3_PORT_DST}" ] && [ $DRY_RUN != "true" ]; then
+ yq4 -i 'del(.networkPolicies.rcloneSync.destinationObjectStorageS3)' "${config["override_sc"]}"
+ else
+ checkIfDiffAndUpdateDNSIPs "${S3_ENDPOINT_DST}" ".networkPolicies.rcloneSync.destinationObjectStorageS3.ips" "${config["override_sc"]}"
+ checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.destinationObjectStorageS3.ports" "${config["override_sc"]}" "$S3_PORT_DST"
+ fi
+
+ checkIfDiffAndUpdateDNSIPs "${SWIFT_ENDPOINT_DST}" ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ips" "${config["override_sc"]}"
+ checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ports" "${config["override_sc"]}" "$SWIFT_PORT_DST"
+ ifNull=true
 
- S3_ENDPOINT_DST="$(yq_dig 'sc' '.objectStorage.sync.s3.regionEndpoint' '""' | sed 's/https\?:\/\///' | sed 's/[:\/].*//')"
- if [[ "${S3_ENDPOINT_DST}" == "" ]]; then
- log_error "No destination S3 endpoint for rclone sync found, check your sc-config.yaml"
- exit 1
  fi
- S3_PORT_DST="$(yq_dig 'sc' '.objectStorage.sync.s3.regionEndpoint' '""' | sed 's/https\?:\/\///' | sed 's/[A-Za-z.0-9-]*:\?//' | sed 's/\/.*//')"
- if [ -z "$S3_PORT_DST" ]; then
- S3_PORT_DST="443"
+ if { [ "$destinationSwift" == "true" ] && [ "$destinationS3" == "true" ]; } || [ -z "$ifNull" ] && { [ "$(yq_dig 'sc' '.objectStorage.sync.type' 'swift')" == "swift" ] || [ "$(yq_dig 'sc' '.objectStorage.sync.type' 's3')" == "s3" ]; }; then
+ if [ -z "${S3_ENDPOINT_DST}" ]; then
+ log_error "No destination S3 endpoint for rclone sync found, check your sc-config.yaml"
+ exit 1
+ fi
+ if [ -z "${SWIFT_ENDPOINT_DST}" ]; then
+ log_error "No destination Swift endpoint for rclone sync found, check your sc-config.yaml"
+ exit 1
+ fi
+ if [ -z "${S3_PORT_DST}" ]; then
+ S3_PORT_DST="443"
+ fi
+ if [ -z "${SWIFT_PORT_DST}" ]; then
+ SWIFT_PORT_DST="443"
+ fi
+
+ checkIfDiffAndUpdateDNSIPs "${S3_ENDPOINT_DST}" ".networkPolicies.rcloneSync.destinationObjectStorageS3.ips" "${config["override_sc"]}"
+ checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.destinationObjectStorageS3.ports" "${config["override_sc"]}" "$S3_PORT_DST"
+
+ checkIfDiffAndUpdateDNSIPs "${SWIFT_ENDPOINT_DST}" ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ips" "${config["override_sc"]}"
+ checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.destinationObjectStorageSwift.ports" "${config["override_sc"]}" "$SWIFT_PORT_DST"
  fi
 
- checkIfDiffAndUpdateDNSIPs "${S3_ENDPOINT_DST}" ".networkPolicies.rcloneSync.destinationObjectStorage.ips" "${config["override_sc"]}"
-
- checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.destinationObjectStorage.ports" "${config["override_sc"]}" "$S3_PORT_DST"
+ SECONDARY_ENDPOINT="$(yq_dig 'sc' '.objectStorage.sync.secondaryUrl' "" | sed 's/https\?:\/\///' | sed 's/[:\/].*//')"
+ if [ -n "${SECONDARY_ENDPOINT}" ]; then
+ SECONDARY_PORT="$(yq_dig 'sc' '.objectStorage.sync.secondaryUrl' "" | sed 's/https\?:\/\///' | sed 's/[A-Za-z.0-9-]*:\?//' | sed 's/\/.*//')"
+ if [ -z "${SECONDARY_PORT}" ]; then
+ SECONDARY_PORT="443"
+ fi
+ checkIfDiffAndUpdateDNSIPs "${SECONDARY_ENDPOINT}" ".networkPolicies.rcloneSync.secondaryUrl.ips" "${config["override_sc"]}"
+ checkIfDiffAndUpdatePorts ".networkPolicies.rcloneSync.secondaryUrl.ports" "${config["override_sc"]}" "$SECONDARY_PORT"
+
+ elif [ -z "${SECONDARY_ENDPOINT}" ] && [ $DRY_RUN == "true" ]; then
+ results_diff=$(diff -U0 --color=always <(yq4 -P 'sort_keys(.networkPolicies.rcloneSync.secondaryUrl)' "${config["override_sc"]}") <(yq4 -P 'del(.networkPolicies.rcloneSync.secondaryUrl)' "${config["override_sc"]}") --label "${config["override_sc"]//${CK8S_CONFIG_PATH}\//}" --label expected) || true
+ if [ "${results_diff}" != "" ]; then
+ printf "${results_diff}"'%s\n'
+ log_warning "Diff found for .networkPolicies.rcloneSync.secondaryUrl in ${config[override_sc]//${CK8S_CONFIG_PATH}\//} (diff shows actions needed to be up to date)"
+ fi
+ elif [ -z "${SECONDARY_ENDPOINT}" ] && [ $DRY_RUN != "true" ]; then
+ yq4 -i 'del(.networkPolicies.rcloneSync.secondaryUrl)' "${config["override_sc"]}"
+ fi
  fi
 fi
 

config/config/sc-config.yaml

@@ -9,7 +9,7 @@ global:
  - ${CK8S_ENVIRONMENT_NAME}-wc
 
 objectStorage:
- ## Swift can be enabled separately for Harbor and Thanos only.
+ ## Swift can be enabled separately for Harbor, Thanos and rclone-sync only.
  # swift:
  # authVersion: 0 # auto detect
  # authUrl: set-me
@@ -32,15 +32,19 @@ objectStorage:
  enabled: false
  # dryrun: false
 
- ## Options are 's3'
+ ## Options are 's3 and swift'
  type: none
+ # secondaryUrl: set-me if regionEndpoint and or authUrl does not have all the relevant ips and or ports used for rclone-sync networkpolicy.
  # s3:
  # region: set-me
  # regionEndpoint: set-me
  # # Generally false when using AWS and Exoscale and true for other providers.
  # forcePathStyle: set-me
  # # v2Auth: false
-
+ ## rclone-sync needs authUrl with a /v3 suffix for swift.
+ # swift:
+ # authUrl: set-me
+ # region: set-me
  ## Sync all buckets under 'objectStorage.buckets'
  ## These will be appended to 'buckets' using the same name from source as destination, and the default schedule.
  syncDefaultBuckets: false
@@ -53,7 +57,8 @@ objectStorage:
  # - source: source-bucket
  # # destination: destination-bucket # if unset it will be the same as 'source'
  # # schedule: 0 5 * * * # if unset it will be the same as 'defaultSchedule'
-
+ # # sourceType: Optional but if included it uses s3 or swift
+ # # destinationType: Optional but if included it uses s3 or swift.
  # Encrypt (symmetric) before syncing
  encrypt:
  enabled: false
@@ -1107,10 +1112,8 @@ welcomingDashboard:
 networkPolicies:
  global:
  objectStorageSwift:
- ips:
- - set-me-if-enabled
- ports:
- - set-me-if-enabled
+ ips: [] # - set-me-if-enabled
+ ports: [] # - set-me-if-enabled
  scApiserver:
  # usually private ip of control-plane nodes
  ips:
@@ -1205,12 +1208,15 @@ networkPolicies:
 
  rcloneSync:
  enabled: true
- destinationObjectStorage:
- ips:
- - "set-me-if-objectStorage.sync.enabled"
- ports:
- - 443
-
+ destinationObjectStorageS3:
+ ips: [] # - "set-me-if-objectStorage.sync.enabled-and-type-is-s3"
+ ports: [] # - 443
+ destinationObjectStorageSwift:
+ ips: [] # - "set-me-if-objectStorage.sync.enabled-and-type-is-swift"
+ ports: [] # - 443
+ secondaryUrl:
+ ips: [] # - "set-me-if-secondaryUrl-has-a-url"
+ ports: [] # - 443
  s3Exporter:
  enabled: true
 

config/secrets/sc-secrets.yaml

@@ -8,7 +8,7 @@ objectStorage: {}
  # swift:
  # username: set-me
  # password: set-me
- # # Configure application credential to use with Thanos
+ # # Configure application credential to use with Thanos and or rclone-sync
  # applicationCredentialID: "set-me"
  # applicationCredentialSecret: "set-me"
  #
@@ -17,6 +17,9 @@ objectStorage: {}
  # s3:
  # accessKey: "set-me"
  # secretKey: "set-me"
+ # swift:
+ # applicationCredentialID: "set-me"
+ # applicationCredentialSecret: "set-me"
  # encrypt:
  # password: "set-me" # generate with `pwgen 32 1`
  # salt: "set-me" # generate with `pwgen 32 1`

helmfile/charts/networkpolicy/service-cluster/templates/rclone-sync.yaml

@@ -18,23 +18,23 @@ spec:
  protocol: TCP
  {{- if and .Values.global.objectStorage.ips .Values.global.objectStorage.ports }}
  - to:
- {{- range .Values.global.objectStorage.ips }}
+ {{- range .Values.global.objectStorage.ips | uniq }}
  - ipBlock:
  cidr: {{ . }}
  {{- end }}
  ports:
- {{- range .Values.global.objectStorage.ports }}
+ {{- range .Values.global.objectStorage.ports | uniq }}
  - port: {{ . }}
  {{- end }}
  {{- end }}
  {{- if and .Values.rcloneSync.destinationObjectStorage.ips .Values.rcloneSync.destinationObjectStorage.ports }}
  - to:
- {{- range .Values.rcloneSync.destinationObjectStorage.ips }}
+ {{- range .Values.rcloneSync.destinationObjectStorage.ips | uniq }}
  - ipBlock:
  cidr: {{ . }}
  {{- end }}
  ports:
- {{- range .Values.rcloneSync.destinationObjectStorage.ports }}
+ {{- range .Values.rcloneSync.destinationObjectStorage.ports | uniq }}
  - port: {{ . }}
  {{- end }}
  {{- end }}

helmfile/charts/rclone-sync/files/rclone.conf

@@ -1,23 +1,21 @@
-[{{ .Values.config.source.name }}]
-type = {{ .Values.config.source.type }}
-{{- if eq .Values.config.source.type "s3" }}
-access_key_id = {{ .Values.config.source.s3.accessKey }}
-secret_access_key = {{ .Values.config.source.s3.secretKey }}
-region = {{ .Values.config.source.s3.region }}
-endpoint = {{ .Values.config.source.s3.regionEndpoint }}
-force_path_style = {{ .Values.config.source.s3.forcePathStyle }}
-v2_auth = {{ .Values.config.source.s3.v2Auth }}
+{{- range concat .Values.config.source .Values.config.destination }}
+[{{ .name }}]
+type = {{ .type }}
+{{- if eq .type "s3" }}
+access_key_id = {{ .s3.accessKey }}
+secret_access_key = {{ .s3.secretKey }}
+region = {{ .s3.region }}
+endpoint = {{ .s3.regionEndpoint }}
+force_path_style = {{ .s3.forcePathStyle }}
+v2_auth = {{ .s3.v2Auth | default false }}
 {{- end }}
 
-[{{ .Values.config.destination.name }}]
-type = {{ .Values.config.destination.type }}
-{{- if eq .Values.config.destination.type "s3" }}
-access_key_id = {{ .Values.config.destination.s3.accessKey }}
-secret_access_key = {{ .Values.config.destination.s3.secretKey }}
-region = {{ .Values.config.destination.s3.region }}
-endpoint = {{ .Values.config.destination.s3.regionEndpoint }}
-force_path_style = {{ .Values.config.destination.s3.forcePathStyle }}
-v2_auth = {{.Values.config.destination.s3.v2Auth }}
+{{- if eq .type "swift" }}
+application_credential_id = {{ .swift.applicationCredentialID }}
+application_credential_secret = {{ .swift.applicationCredentialSecret }}
+auth = {{ .swift.authUrl }}
+region = {{ .swift.region }}
+{{- end }}
 {{- end }}
 
 {{- if .Values.config.encrypt.enabled }}

helmfile/charts/rclone-sync/templates/cronjob.yaml

@@ -29,11 +29,11 @@ spec:
  image: {{ $.Values.image.repository }}:{{ $.Values.image.tag }}
  args:
  - sync
- - {{ $.Values.config.source.name }}:{{ .source }}
+ - src-{{ .sourceType }}:{{ .source }}
  {{- if $.Values.config.encrypt.enabled }}
  - "{{ $.Values.config.encrypt.name }}-{{ .destination }}:"
  {{- else }}
- - {{ $.Values.config.destination.name }}:{{ .destination }}
+ - dest-{{ .destinationType }}:{{ .destination }}
  {{- end }}
  - --log-level
  - DEBUG

helmfile/charts/rclone-sync/values.yaml

@@ -6,7 +6,7 @@ config:
  dryrun: false
 
  source:
-   name: provider-region-a
+ - name: provider-region-a
  type: s3
  s3:
  region: region
@@ -15,9 +15,16 @@ config:
  secretKey: secret-key
  forcePathStyle: true
  v2Auth: false
+ - name: provider-region-a
+ type: swift
+ swift:
+ applicationCredentialID: access-key
+ applicationCredentialSecret: secret-key
+ authUrl: swift.region.provider.net
+ region: region
 
  destination:
-   name: provider-region-b
+ - name: provider-region-b
  type: s3
  s3:
  region: region
@@ -26,6 +33,13 @@ config:
  secretKey: secret-key
  forcePathStyle: true
  v2Auth: false
+ - name: provider-region-b
+ type: swift
+ swift:
+ applicationCredentialID: access-key
+ applicationCredentialSecret: secret-key
+ authUrl: swift.region.provider.net
+ region: region
 
  encrypt:
  enabled: false
@@ -44,6 +58,8 @@ buckets:
  - source: bucket
  destination: other-bucket
  schedule: 0 5 * * *
+ sourceType: # Optional but if included types are s3 or swift.
+ destinationType: # Optional but if included types are s3 or swift.
 
 resources:
  requests: