feat: introduce maxscale

With the release of the mariadb operator v0.25.0 maxscale was introduced to resolve issues with multi-master deployments, enhance scale, and make better use of nodes in the environment. This change creates the maxscale resources and converts our standard deployment systems to use maxscale as the point of ingress. Specific feature we're interested in > point to an existing MaxScale instance. Doing this will delegate tasks such as primary failover to MaxScale. Docs: https://github.com/mariadb-operator/mariadb-operator/blob/main/docs/MAXSCALE.md Related: https://github.com/mariadb-operator/mariadb-operator/releases/tag/v0.0.26 Signed-off-by: Kevin Carter <[email protected]>
rackerlabs · Mar 21, 2024 · f0f4157 · f0f4157
1 parent d49012c
commit f0f4157
Show file tree

Hide file tree

Showing 19 changed files with 179 additions and 38 deletions.
diff --git a/docs/infrastructure-mariadb-connect.md b/docs/infrastructure-mariadb-connect.md
@@ -3,9 +3,9 @@
 Sometimes an operator may need to connect to the database to troubleshoot things or otherwise make modifications to the databases in place. The following command can be used to connect to the database from a node within the cluster.
 
 ``` shell
-mysql -h $(kubectl -n openstack get service mariadb-galera-primary -o jsonpath='{.spec.clusterIP}') \
-      -p$(kubectl --namespace openstack get secret mariadb -o jsonpath='{.data.root-password}' | base64 -d) \
-      -u root
+mysql -h $(kubectl -n openstack get service maxscale-galera -o jsonpath='{.spec.clusterIP}') \
+      -p$(kubectl --namespace openstack get secret maxscale -o jsonpath='{.data.password}' | base64 -d) \
+      -u maxscale-galera-client
 ```
 
 !!! info

diff --git a/docs/infrastructure-mariadb.md b/docs/infrastructure-mariadb.md
@@ -42,3 +42,19 @@ kubectl --namespace openstack apply -k /opt/genestack/kustomize/mariadb-cluster/
 ``` shell
 kubectl --namespace openstack get mariadbs -w
 ```
+
+## MaxScale
+
+Within the deployment the OpenStack services use MaxScale for loadlancing and greater reliability. While the MaxScale ecosystem is a good one, there are some limitations that you should be aware of. It is recommended that you review the [MaxScale reference documentation](https://mariadb.com/kb/en/mariadb-maxscale-2302-limitations-and-known-issues-within-mariadb-maxscale) for more about all of the known limitations and potential workarounds available.
+
+``` mermaid
+flowchart TD
+    A[Connection] ---B{MaxScale}
+    B ---|ro| C[ES-0]
+    B ---|rw| D[ES-1] ---|sync| E & C
+    B ---|ro| E[ES-2]
+```
+
+### MaxScale GUI
+
+The MaxScale deployment has access to a built in GUI that can be exposed for further debuging and visibility into the performance of the MariDB backend. For more information on accessing the GUI please refer to the MaxScale documentation that can be found [here](https://mariadb.com/resources/blog/getting-started-with-the-mariadb-maxscale-gui).
diff --git a/docs/openstack-skyline.md b/docs/openstack-skyline.md
@@ -17,7 +17,7 @@ kubectl --namespace openstack \
         --from-literal=service-domain="service" \
         --from-literal=service-project="service" \
         --from-literal=service-project-domain="service" \
-        --from-literal=db-endpoint="mariadb-galera-primary.openstack.svc.cluster.local" \
+        --from-literal=db-endpoint="maxscale-galera.openstack.svc.cluster.local" \
         --from-literal=db-name="skyline" \
         --from-literal=db-username="skyline" \
         --from-literal=db-password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)" \

diff --git a/helm-configs/cinder/cinder-helm-overrides.yaml b/helm-configs/cinder/cinder-helm-overrides.yaml
@@ -1320,7 +1320,7 @@ endpoints:
         username: cinder
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /cinder

diff --git a/helm-configs/glance/glance-helm-overrides.yaml b/helm-configs/glance/glance-helm-overrides.yaml
@@ -589,7 +589,7 @@ endpoints:
         username: glance
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /glance

diff --git a/helm-configs/gnocchi/gnocchi-helm-overrides.yaml b/helm-configs/gnocchi/gnocchi-helm-overrides.yaml
@@ -622,7 +622,7 @@ endpoints:
         username: gnocchi
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /gnocchi

diff --git a/helm-configs/heat/heat-helm-overrides.yaml b/helm-configs/heat/heat-helm-overrides.yaml
@@ -859,7 +859,7 @@ endpoints:
         username: heat
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /heat

diff --git a/helm-configs/horizon/horizon-helm-overrides.yaml b/helm-configs/horizon/horizon-helm-overrides.yaml
@@ -7242,7 +7242,7 @@ endpoints:
         username: horizon
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /horizon

diff --git a/helm-configs/keystone/keystone-helm-overrides.yaml b/helm-configs/keystone/keystone-helm-overrides.yaml
@@ -972,7 +972,7 @@ endpoints:
         username: keystone
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /keystone

diff --git a/helm-configs/neutron/neutron-helm-overrides.yaml b/helm-configs/neutron/neutron-helm-overrides.yaml
@@ -2199,7 +2199,7 @@ endpoints:
         username: neutron
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /neutron

diff --git a/helm-configs/nova/nova-helm-overrides.yaml b/helm-configs/nova/nova-helm-overrides.yaml
@@ -1640,7 +1640,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /nova
@@ -1657,7 +1657,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /nova_api
@@ -1674,7 +1674,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /nova_cell0

diff --git a/helm-configs/octavia/octavia-helm-overrides.yaml b/helm-configs/octavia/octavia-helm-overrides.yaml
@@ -466,7 +466,7 @@ endpoints:
         username: octavia
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /octavia

diff --git a/helm-configs/placement/placement-helm-overrides.yaml b/helm-configs/placement/placement-helm-overrides.yaml
@@ -206,7 +206,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /placement

diff --git a/kustomize/grafana/base/grafana-database.yaml b/kustomize/grafana/base/grafana-database.yaml
@@ -1,4 +1,4 @@
-apiVersion: mariadb.mmontes.io/v1alpha1
+apiVersion: k8s.mariadb.com/v1alpha1
 kind: Database
 metadata:
   name: grafana
@@ -13,7 +13,7 @@ spec:
   requeueInterval: 30s
   retryInterval: 5s
 ---
-apiVersion: mariadb.mmontes.io/v1alpha1
+apiVersion: k8s.mariadb.com/v1alpha1
 kind: User
 metadata:
   name: grafana
@@ -32,7 +32,7 @@ spec:
   requeueInterval: 30s
   retryInterval: 5s
 ---
-apiVersion: mariadb.mmontes.io/v1alpha1
+apiVersion: k8s.mariadb.com/v1alpha1
 kind: Grant
 metadata:
   name: grant

diff --git a/kustomize/mariadb-cluster/aio/kustomization.yaml b/kustomize/mariadb-cluster/aio/kustomization.yaml
@@ -12,3 +12,10 @@ patches:
       - op: replace
         path: /spec/galera/enabled
         value: false
+  - target:
+      kind: MaxScale
+      name: maxscale-galera
+    patch: |-
+      - op: replace
+        path: /spec/replicas
+        value: 1
diff --git a/kustomize/mariadb-cluster/base/kustomization.yaml b/kustomize/mariadb-cluster/base/kustomization.yaml
@@ -1,4 +1,5 @@
 resources:
   - mariadb-configmap.yaml
+  - mariadb-maxscale.yaml
   - mariadb-galera.yaml
   - mariadb-backup.yaml
diff --git a/kustomize/mariadb-cluster/base/mariadb-galera.yaml b/kustomize/mariadb-cluster/base/mariadb-galera.yaml
@@ -28,21 +28,8 @@ spec:
     runAsUser: 0
 
   # point to an existing MaxScale instance. Doing this will delegate tasks such as primary failover to MaxScale.
-  # maxScaleRef:
-  #   name: maxscale
-
-  # provision a MaxScale instance and set 'spec.maxScaleRef' automatically.
-  maxScale:
-    enabled: false
-
-    kubernetesService:
-      type: LoadBalancer
-      annotations:
-        metallb.universe.tf/address-pool: primary
-
-    connection:
-      secretName: mxs-galera-conn
-      port: 3306
+  maxScaleRef:
+    name: maxscale-galera
 
   galera:
     enabled: true
@@ -132,8 +119,6 @@ spec:
   resources:
     requests:
       memory: 256Mi
-    limits:
-      memory: 16Gi
 
   metrics:
     enabled: true
diff --git a/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml b/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml
@@ -0,0 +1,132 @@
+apiVersion: k8s.mariadb.com/v1alpha1
+kind: MaxScale
+metadata:
+  name: maxscale-galera
+spec:
+  replicas: 3
+
+  mariaDbRef:
+    name: mariadb-galera
+    namespace: openstack
+
+  services:
+    - name: rw-router
+      router: readwritesplit
+      params:
+        transaction_replay: "true"
+        transaction_replay_attempts: "10"
+        transaction_replay_timeout: "5s"
+        max_slave_connections: "255"
+        max_replication_lag: "3s"
+        master_accept_reads: "true"
+      listener:
+        name: rw-listener
+        port: 3306
+        protocol: MariaDBProtocol
+        params:
+          connection_metadata: "tx_isolation=auto"
+        suspend: false
+      suspend: false
+    - name: rconn-master-router
+      router: readconnroute
+      params:
+        router_options: "master"
+        max_replication_lag: "3s"
+        master_accept_reads: "true"
+      listener:
+        port: 3307
+    - name: rconn-slave-router
+      router: readconnroute
+      params:
+        router_options: "slave"
+        max_replication_lag: "3s"
+      listener:
+        port: 3308
+
+  monitor:
+    name: mariadb-monitor
+    module: galeramon
+    interval: 2s
+    cooperativeMonitoring: majority_of_all
+    params:
+      disable_master_failback: "false"
+      available_when_donor: "false"
+      disable_master_role_setting: "false"
+    suspend: false
+
+  admin:
+    port: 8989
+    guiEnabled: true
+
+  config:
+    params:
+      log_info: "true"
+    volumeClaimTemplate:
+      resources:
+        requests:
+          storage: 100Mi
+      accessModes:
+        - ReadWriteOnce
+      storageClassName: general
+    sync:
+      database: mysql
+      interval: 5s
+      timeout: 10s
+
+  auth:
+    generate: true
+    adminUsername: mariadb-operator
+    adminPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    deleteDefaultAdmin: true
+    clientUsername: maxscale-galera-client
+    clientPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    clientMaxConnections: 1024
+    serverUsername: maxscale-galera-server
+    serverPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    serverMaxConnections: 1024
+    monitorUsername: maxscale-galera-monitor
+    monitorPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    monitorMaxConnections: 128
+    syncUsername: maxscale-galera-sync
+    syncPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    syncMaxConnections: 128
+
+  securityContext:
+    allowPrivilegeEscalation: false
+
+  updateStrategy:
+    type: RollingUpdate
+
+  kubernetesService:
+    type: LoadBalancer
+    annotations:
+      metallb.universe.tf/address-pool: primary
+
+  connection:
+    secretName: mxs-galera-conn
+    port: 3306
+
+  resources:
+    requests:
+      memory: 128Mi
+
+  affinity:
+    enableAntiAffinity: true
+
+  tolerations:
+    - key: "k8s.mariadb.com/ha"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+  podDisruptionBudget:
+    maxUnavailable: 33%
diff --git a/kustomize/octavia/base/octavia-agent.yaml b/kustomize/octavia/base/octavia-agent.yaml
@@ -81,7 +81,7 @@ spec:
             - name: PATH
               value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/
             - name: DEPENDENCY_SERVICE
-              value: "openstack:mariadb-galera-primary,openstack:keystone-api,openstack:rabbitmq-nodes,openstack:memcached,openstack:neutron-server"
+              value: "openstack:maxscale-galera,openstack:keystone-api,openstack:rabbitmq-nodes,openstack:memcached,openstack:neutron-server"
             - name: DEPENDENCY_JOBS
               value: "octavia-db-sync,octavia-ks-user,octavia-ks-endpoints"
             - name: DEPENDENCY_DAEMONSET
@@ -187,4 +187,4 @@ spec:
             secretName: octavia-etc
             defaultMode: 0444
         - emptyDir: {}
-          name: pod-run-octavia
+          name: pod-run-octavia