S3 upload (#150)

Co-authored-by: Renaud Chaput <[email protected]>

Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time
 # you make changes to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 5.4.1
+version: 5.5.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

templates/job-assets-copy.yaml

@@ -0,0 +1,90 @@
+{{- if .Values.mastodon.hooks.s3Upload.enabled -}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "mastodon.fullname" . }}-assets-upload
+  labels:
+    {{- include "mastodon.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation
+    "helm.sh/hook-weight": "-1"
+spec:
+  template:
+    metadata:
+      name: {{ include "mastodon.fullname" . }}-assets-upload
+      {{- with .Values.jobAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+      restartPolicy: Never
+      initContainers:
+        - name: extract-assets
+          image: "{{ coalesce .Values.mastodon.web.image.repository .Values.image.repository }}:{{ coalesce .Values.mastodon.web.image.tag .Values.image.tag .Chart.AppVersion }}"
+          imagePullPolicy: Always
+          command:
+          - cp
+          args:
+          - -rv
+          - public
+          - /assets
+          volumeMounts:
+            - mountPath: /assets
+              name: assets
+      containers:
+        - name: upload-assets
+          image: rclone/rclone:1
+          imagePullPolicy: Always
+          env:
+          - name: RCLONE_S3_NO_CHECK_BUCKET
+            value: "true"
+          - name: RCLONE_CONFIG_REMOTE_TYPE
+            value: s3
+          - name: RCLONE_CONFIG_REMOTE_PROVIDER
+            value: AWS
+          - name: RCLONE_CONFIG_REMOTE_ENDPOINT
+            value: {{ required "Please specify an endpoint for S3 asset uploads" .Values.mastodon.hooks.s3Upload.endpoint }}
+          - name: RCLONE_CONFIG_REMOTE_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: {{ required "Please specify a secret with S3 credentials for S3 asset uploads" .Values.mastodon.hooks.s3Upload.secretRef.name }}
+                key: {{ .Values.mastodon.hooks.s3Upload.secretRef.keys.accesKeyId }}
+          - name: RCLONE_CONFIG_REMOTE_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: {{ required "Please specify a secret with S3 credentials for S3 asset uploads" .Values.mastodon.hooks.s3Upload.secretRef.name }}
+                key: {{ .Values.mastodon.hooks.s3Upload.secretRef.keys.secretAccessKey }}
+          {{- with .Values.mastodon.hooks.s3Upload.rclone.env }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          command:
+          - rclone
+          args:
+          - copy
+          - /assets/public
+          - "remote:{{ required "Please specify a bucket for S3 asset uploads" .Values.mastodon.hooks.s3Upload.bucket }}"
+          - --fast-list
+          - --transfers=32
+          - --include
+          - "{assets,packs}/**"
+          - --progress
+          - -vv
+          volumeMounts:
+            - mountPath: /assets
+              name: assets
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              memory: 500Mi
+      volumes:
+        - name: assets
+          emptyDir: {}
+{{- end -}}

values.yaml

@@ -28,6 +28,27 @@ mastodon:
       enabled: true
     assetsPrecompile:
       enabled: true
+    # Upload website assets to S3 before deploying using rclone.
+    # Whenever there is an update to Mastodon, sometimes there are assets files
+    # that are renamed. As the pods are getting redeployed, and old/new pods are
+    # present simultaneously, there is a chance that old asset files are
+    # requested from pods that don't have them anymore, or new asset files are
+    # requested from old pods. Uploading asset files to S3 in this manner solves
+    # this potential conflict.
+    # Note that you will need to CDN/proxy to send all requests to /assets and
+    # /packs to this bucket.
+    s3Upload:
+      enabled: false
+      endpoint:
+      bucket:
+      secretRef:
+        name:
+        keys:
+          accesKeyId: acces-key-id
+          secretAccessKey: secret-access-key
+      rclone:
+        # Any additional environment variables to pass to rclone.
+        env: {}
   # Custom labels to add to kubernetes resources
   #labels:
   cron: