feat: add support for embedded etcd

.github/kind.yml

@@ -31,6 +31,9 @@ nodes:
       - containerPort: 443
         hostPort: 443
         protocol: TCP
+    extraMounts:
+      - hostPath: /home/tmp
+        containerPath: /data
   - role: worker
   - role: worker
   - role: worker

.github/workflows/conformance-tests.yaml

@@ -82,7 +82,11 @@ jobs:
       - name: Install latest kyverno
         run: |
           set -e
-          kubectl create -f https://github.com/kyverno/kyverno/raw/main/config/install-latest-testing.yaml
+          set -e
+          export HELM=${{ steps.helm.outputs.helm-path }}
+          helm repo add kyverno https://kyverno.github.io/kyverno/
+          kubectl create namespace kyverno
+          helm install kyverno --namespace kyverno kyverno/kyverno --set features.policyExceptions.enabled=true --set features.policyExceptions.namespace='*'
       - name: Wait for kyverno ready
         run: |
           set -e

.github/workflows/migration-tests.yaml

@@ -70,10 +70,13 @@ jobs:
         run: |
           set -e
           kind create cluster --image kindest/node:${{ matrix.k8s-version.version }} --config ./.github/kind.yml
-      - name: Install kyverno v1.12.4
+      - name: Install kyverno
         run: |
           set -e
-          kubectl create -f https://github.com/kyverno/kyverno/raw/main/config/install-latest-testing.yaml
+          export HELM=${{ steps.helm.outputs.helm-path }}
+          helm repo add kyverno https://kyverno.github.io/kyverno/
+          kubectl create namespace kyverno
+          helm install kyverno --namespace kyverno kyverno/kyverno --set features.policyExceptions.enabled=true --set features.policyExceptions.namespace='*'
       - name: Wait for kyverno ready
         run: |
           set -e

Makefile

@@ -164,23 +164,23 @@ codegen-install-manifest: $(HELM) ## Create install manifest
  		| $(SED) -e '/^#.*/d' \
 		> ./config/install.yaml
 
-codegen-install-manifest-inmemory: $(HELM) ## Create install manifest without postgres
+codegen-install-manifest-etcd: $(HELM) ## Create install manifest without postgres
 	@echo Generate latest install manifest... >&2
 	@$(HELM) template reports-server --namespace reports-server ./charts/reports-server/ \
 		--set apiServicesManagement.installApiServices.enabled=true \
 		--set image.tag=latest \
-		--set config.debug=true \
+		--set config.etcd.enabled=true \
 		--set postgresql.enabled=false \
 		--set templating.enabled=true \
  		| $(SED) -e '/^#.*/d' \
-		> ./config/install-inmemory.yaml
+		> ./config/install-etcd.yaml
 
 .PHONY: codegen
 codegen: ## Rebuild all generated code and docs
 codegen: codegen-helm-docs
 codegen: codegen-openapi
 codegen: codegen-install-manifest
-codegen: codegen-install-manifest-inmemory
+codegen: codegen-install-manifest-etcd
 
 .PHONY: verify-codegen
 verify-codegen: codegen ## Verify all generated code and docs are up to date
@@ -220,12 +220,12 @@ kind-install: $(HELM) kind-load ## Build image, load it in kind cluster and depl
 		--set image.repository=$(PACKAGE) \
 		--set image.tag=$(GIT_SHA)
 
-.PHONY: kind-install-inmemory
-kind-install-inmemory: $(HELM) kind-load ## Build image, load it in kind cluster and deploy helm chart
+.PHONY: kind-install-etcd
+kind-install-etcd: $(HELM) kind-load ## Build image, load it in kind cluster and deploy helm chart
 	@echo Install chart... >&2
 	@$(HELM) upgrade --install reports-server --namespace reports-server --create-namespace --wait ./charts/reports-server \
 		--set image.registry=$(KO_REGISTRY) \
-		--set config.debug=true \
+		--set config.etcd.enabled=true \
 		--set postgresql.enabled=false \
 		--set image.repository=$(PACKAGE) \
 		--set image.tag=$(GIT_SHA)

charts/reports-server/README.md

@@ -61,7 +61,9 @@ helm install reports-server --namespace reports-server --create-namespace report
 | affinity | object | `{}` | Affinity |
 | service.type | string | `"ClusterIP"` | Service type |
 | service.port | int | `443` | Service port |
-| config.debug | bool | `false` | Enable debug (to use inmemorydatabase) |
+| config.etcd.enabled | bool | `false` |  |
+| config.etcd.endpoints | string | `nil` |  |
+| config.etcd.insecure | bool | `true` |  |
 | config.db.secretName | string | `""` | If set, database connection information will be read from the Secret with this name. Overrides `db.host`, `db.name`, `db.user`, and `db.password`. |
 | config.db.host | string | `""` | Database host |
 | config.db.hostSecretKeyName | string | `"host"` | The database host will be read from this `key` in the specified Secret, when `db.secretName` is set. |

charts/reports-server/templates/deployment.yaml

@@ -37,8 +37,12 @@ spec:
       containers:
         - name: reports-server
           args:
-            {{- if .Values.config.debug }}
-            - --debug
+            {{- if .Values.config.etcd.enabled }}
+            - --etcd
+            {{- if .Values.config.etcd.insecure }}
+            - --etcdSkipTLS
+            {{- end }}
+            - --etcdEndpoints=https://etcd-0.etcd.{{ $.Release.Namespace }}:2379,https://etcd-1.etcd.{{ $.Release.Namespace }}:2379,https://etcd-2.etcd.{{ $.Release.Namespace }}:2379
             {{- else }}
             - --dbhost={{ include "reports-server.dbHost" . }}
             - --dbport={{ include "reports-server.dbPort" . }}
@@ -85,15 +89,15 @@ spec:
           {{- end}}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
+          volumeMounts:
+            - mountPath: /tmp
+              name: tmp-dir
           image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           ports:
             - name: https
               containerPort: 4443
               protocol: TCP
-          volumeMounts:
-            - mountPath: /tmp
-              name: tmp-dir
           {{- with .Values.livenessProbe }}
           livenessProbe:
             {{- toYaml . | nindent 12 }}

charts/reports-server/templates/etcd.yaml

@@ -0,0 +1,170 @@
+{{- if .Values.config.etcd.enabled }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: etcd
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: etcd-reports-server
+    {{- include "reports-server.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  selector:
+    app: etcd-reports-server
+  publishNotReadyAddresses: true
+  ports:
+  - name: etcd-client
+    port: 2379
+  - name: etcd-server
+    port: 2380
+  - name: etcd-metrics
+    port: 8080
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  namespace: {{ include "reports-server.fullname" . }}
+  name: etcd
+  labels:
+    app: etcd-reports-server
+    {{- include "reports-server.labels" . | nindent 4 }}
+spec:
+  serviceName: etcd
+  replicas: 3
+  podManagementPolicy: Parallel
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: etcd-reports-server
+  template:
+    metadata:
+      labels:
+        app: etcd-reports-server
+      annotations:
+        serviceName: etcd
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 1
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - etcd-reports-server
+              topologyKey: "kubernetes.io/hostname"
+      containers:
+      - name: etcd
+        image: quay.io/coreos/etcd:v3.5.15
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: etcd-client
+          containerPort: 2379
+        - name: etcd-server
+          containerPort: 2380
+        - name: etcd-metrics
+          containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8080
+          initialDelaySeconds: 10
+          periodSeconds: 5
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 30
+        livenessProbe:
+          httpGet:
+            path: /livez
+            port: 8080
+          initialDelaySeconds: 15
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
+        env:
+        - name: K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+             fieldPath: metadata.namespace
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+             fieldPath: metadata.name
+        - name: SERVICE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.annotations['serviceName']
+        - name: ETCDCTL_ENDPOINTS
+          value: $(HOSTNAME).$(SERVICE_NAME):2379
+        ## TLS client configuration for etcdctl in the container.
+        ## These files paths are part of the "etcd-client-certs" volume mount.
+        # - name: ETCDCTL_KEY
+        #   value: /etc/etcd/certs/client/tls.key
+        # - name: ETCDCTL_CERT
+        #   value: /etc/etcd/certs/client/tls.crt
+        # - name: ETCDCTL_CACERT
+        #   value: /etc/etcd/certs/client/ca.crt
+        ##
+        ## Use this URI_SCHEME value for non-TLS clusters.
+        - name: URI_SCHEME
+          value: "http"
+        ## TLS: Use this URI_SCHEME for TLS clusters.
+        # - name: URI_SCHEME
+        # value: "https"
+        command:
+        - /usr/local/bin/etcd
+        args:
+        - --name=$(HOSTNAME)
+        - --data-dir=/data
+        - --wal-dir=/data/wal
+        - --listen-peer-urls=$(URI_SCHEME)://0.0.0.0:2380
+        - --listen-client-urls=$(URI_SCHEME)://0.0.0.0:2379
+        - --advertise-client-urls=$(URI_SCHEME)://$(HOSTNAME).$(SERVICE_NAME):2379
+        - --initial-cluster-state=new
+        - --initial-cluster-token=etcd-$(K8S_NAMESPACE)
+        - --initial-cluster=etcd-0=$(URI_SCHEME)://etcd-0.$(SERVICE_NAME):2380,etcd-1=$(URI_SCHEME)://etcd-1.$(SERVICE_NAME):2380,etcd-2=$(URI_SCHEME)://etcd-2.$(SERVICE_NAME):2380
+        - --initial-advertise-peer-urls=$(URI_SCHEME)://$(HOSTNAME).$(SERVICE_NAME):2380
+        - --listen-metrics-urls=http://0.0.0.0:8080
+        # - --auto-compaction-mode=periodic
+        # - --auto-compaction-retention=10m
+        # - --client-cert-auth
+        # - --trusted-ca-file=$(ETCDCTL_CACERT)
+        # - --cert-file=$(ETCDCTL_CERT)
+        # - --key-file=$(ETCDCTL_KEY)
+        # - --peer-client-cert-auth
+        # - --peer-trusted-ca-file=/etc/etcd/certs/server/ca.crt
+        # - --peer-cert-file=/etc/etcd/certs/server/tls.crt
+        # - --peer-key-file=/etc/etcd/certs/server/tls.key
+        volumeMounts:
+        - name: etcd-data
+          mountPath: /data
+        # - name: etcd-client-tls
+        #   mountPath: "/etc/etcd/certs/client"
+        #   readOnly: true
+        # - name: etcd-server-tls
+        #   mountPath: "/etc/etcd/certs/server"
+        #   readOnly: true
+      volumes:
+      # - name: etcd-client-tls
+      #   secret:
+      #     secretName: etcd-client-tls
+      #     optional: false
+      # - name: etcd-server-tls
+      #   secret:
+      #     secretName: etcd-server-tls
+      #     optional: false
+  volumeClaimTemplates:
+  - metadata:
+      name: etcd-data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 1Gi
+{{- end }}
+

charts/reports-server/values.yaml

@@ -162,8 +162,10 @@ service:
 
 config:
 
-  # -- Enable debug (to use inmemorydatabase)
-  debug: false
+  etcd:
+    enabled: false
+    endpoints: ~
+    insecure: true
 
   db:
     # -- If set, database connection information will be read from the Secret with this name. Overrides `db.host`, `db.name`, `db.user`, and `db.password`.