From f0f4157e85ae8286a6e524261cb5e44da4908179 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Tue, 19 Mar 2024 20:07:41 -0500 Subject: [PATCH] feat: introduce maxscale With the release of the mariadb operator v0.25.0 maxscale was introduced to resolve issues with multi-master deployments, enhance scale, and make better use of nodes in the environment. This change creates the maxscale resources and converts our standard deployment systems to use maxscale as the point of ingress. Specific feature we're interested in > point to an existing MaxScale instance. Doing this will delegate tasks such as primary failover to MaxScale. Docs: https://github.com/mariadb-operator/mariadb-operator/blob/main/docs/MAXSCALE.md Related: https://github.com/mariadb-operator/mariadb-operator/releases/tag/v0.0.26 Signed-off-by: Kevin Carter --- docs/infrastructure-mariadb-connect.md | 6 +- docs/infrastructure-mariadb.md | 16 +++ docs/openstack-skyline.md | 2 +- .../cinder/cinder-helm-overrides.yaml | 2 +- .../glance/glance-helm-overrides.yaml | 2 +- .../gnocchi/gnocchi-helm-overrides.yaml | 2 +- helm-configs/heat/heat-helm-overrides.yaml | 2 +- .../horizon/horizon-helm-overrides.yaml | 2 +- .../keystone/keystone-helm-overrides.yaml | 2 +- .../neutron/neutron-helm-overrides.yaml | 2 +- helm-configs/nova/nova-helm-overrides.yaml | 6 +- .../octavia/octavia-helm-overrides.yaml | 2 +- .../placement/placement-helm-overrides.yaml | 2 +- kustomize/grafana/base/grafana-database.yaml | 6 +- .../mariadb-cluster/aio/kustomization.yaml | 7 + .../mariadb-cluster/base/kustomization.yaml | 1 + .../mariadb-cluster/base/mariadb-galera.yaml | 19 +-- .../base/mariadb-maxscale.yaml | 132 ++++++++++++++++++ kustomize/octavia/base/octavia-agent.yaml | 4 +- 19 files changed, 179 insertions(+), 38 deletions(-) create mode 100644 kustomize/mariadb-cluster/base/mariadb-maxscale.yaml diff --git a/docs/infrastructure-mariadb-connect.md b/docs/infrastructure-mariadb-connect.md index 76f2c9e8..7f00c951 100644 --- a/docs/infrastructure-mariadb-connect.md +++ b/docs/infrastructure-mariadb-connect.md @@ -3,9 +3,9 @@ Sometimes an operator may need to connect to the database to troubleshoot things or otherwise make modifications to the databases in place. The following command can be used to connect to the database from a node within the cluster. ``` shell -mysql -h $(kubectl -n openstack get service mariadb-galera-primary -o jsonpath='{.spec.clusterIP}') \ - -p$(kubectl --namespace openstack get secret mariadb -o jsonpath='{.data.root-password}' | base64 -d) \ - -u root +mysql -h $(kubectl -n openstack get service maxscale-galera -o jsonpath='{.spec.clusterIP}') \ + -p$(kubectl --namespace openstack get secret maxscale -o jsonpath='{.data.password}' | base64 -d) \ + -u maxscale-galera-client ``` !!! info diff --git a/docs/infrastructure-mariadb.md b/docs/infrastructure-mariadb.md index 0f9bfbcb..28aa5539 100644 --- a/docs/infrastructure-mariadb.md +++ b/docs/infrastructure-mariadb.md @@ -42,3 +42,19 @@ kubectl --namespace openstack apply -k /opt/genestack/kustomize/mariadb-cluster/ ``` shell kubectl --namespace openstack get mariadbs -w ``` + +## MaxScale + +Within the deployment the OpenStack services use MaxScale for loadlancing and greater reliability. While the MaxScale ecosystem is a good one, there are some limitations that you should be aware of. It is recommended that you review the [MaxScale reference documentation](https://mariadb.com/kb/en/mariadb-maxscale-2302-limitations-and-known-issues-within-mariadb-maxscale) for more about all of the known limitations and potential workarounds available. + +``` mermaid +flowchart TD + A[Connection] ---B{MaxScale} + B ---|ro| C[ES-0] + B ---|rw| D[ES-1] ---|sync| E & C + B ---|ro| E[ES-2] +``` + +### MaxScale GUI + +The MaxScale deployment has access to a built in GUI that can be exposed for further debuging and visibility into the performance of the MariDB backend. For more information on accessing the GUI please refer to the MaxScale documentation that can be found [here](https://mariadb.com/resources/blog/getting-started-with-the-mariadb-maxscale-gui). diff --git a/docs/openstack-skyline.md b/docs/openstack-skyline.md index 761dcd6e..588bff41 100644 --- a/docs/openstack-skyline.md +++ b/docs/openstack-skyline.md @@ -17,7 +17,7 @@ kubectl --namespace openstack \ --from-literal=service-domain="service" \ --from-literal=service-project="service" \ --from-literal=service-project-domain="service" \ - --from-literal=db-endpoint="mariadb-galera-primary.openstack.svc.cluster.local" \ + --from-literal=db-endpoint="maxscale-galera.openstack.svc.cluster.local" \ --from-literal=db-name="skyline" \ --from-literal=db-username="skyline" \ --from-literal=db-password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)" \ diff --git a/helm-configs/cinder/cinder-helm-overrides.yaml b/helm-configs/cinder/cinder-helm-overrides.yaml index 9d15d034..136fa62c 100644 --- a/helm-configs/cinder/cinder-helm-overrides.yaml +++ b/helm-configs/cinder/cinder-helm-overrides.yaml @@ -1320,7 +1320,7 @@ endpoints: username: cinder password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /cinder diff --git a/helm-configs/glance/glance-helm-overrides.yaml b/helm-configs/glance/glance-helm-overrides.yaml index b3b19a86..6a5ad87c 100644 --- a/helm-configs/glance/glance-helm-overrides.yaml +++ b/helm-configs/glance/glance-helm-overrides.yaml @@ -589,7 +589,7 @@ endpoints: username: glance password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /glance diff --git a/helm-configs/gnocchi/gnocchi-helm-overrides.yaml b/helm-configs/gnocchi/gnocchi-helm-overrides.yaml index db1c37bb..9110f4e9 100644 --- a/helm-configs/gnocchi/gnocchi-helm-overrides.yaml +++ b/helm-configs/gnocchi/gnocchi-helm-overrides.yaml @@ -622,7 +622,7 @@ endpoints: username: gnocchi password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /gnocchi diff --git a/helm-configs/heat/heat-helm-overrides.yaml b/helm-configs/heat/heat-helm-overrides.yaml index b27640c1..138eb76e 100644 --- a/helm-configs/heat/heat-helm-overrides.yaml +++ b/helm-configs/heat/heat-helm-overrides.yaml @@ -859,7 +859,7 @@ endpoints: username: heat password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /heat diff --git a/helm-configs/horizon/horizon-helm-overrides.yaml b/helm-configs/horizon/horizon-helm-overrides.yaml index 298f8238..4563074f 100644 --- a/helm-configs/horizon/horizon-helm-overrides.yaml +++ b/helm-configs/horizon/horizon-helm-overrides.yaml @@ -7242,7 +7242,7 @@ endpoints: username: horizon password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /horizon diff --git a/helm-configs/keystone/keystone-helm-overrides.yaml b/helm-configs/keystone/keystone-helm-overrides.yaml index 09667ed0..cb6f0481 100644 --- a/helm-configs/keystone/keystone-helm-overrides.yaml +++ b/helm-configs/keystone/keystone-helm-overrides.yaml @@ -972,7 +972,7 @@ endpoints: username: keystone password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /keystone diff --git a/helm-configs/neutron/neutron-helm-overrides.yaml b/helm-configs/neutron/neutron-helm-overrides.yaml index ac8f036f..2e9dbdd0 100644 --- a/helm-configs/neutron/neutron-helm-overrides.yaml +++ b/helm-configs/neutron/neutron-helm-overrides.yaml @@ -2199,7 +2199,7 @@ endpoints: username: neutron password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /neutron diff --git a/helm-configs/nova/nova-helm-overrides.yaml b/helm-configs/nova/nova-helm-overrides.yaml index 450a8a9d..75e07d39 100644 --- a/helm-configs/nova/nova-helm-overrides.yaml +++ b/helm-configs/nova/nova-helm-overrides.yaml @@ -1640,7 +1640,7 @@ endpoints: username: nova password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /nova @@ -1657,7 +1657,7 @@ endpoints: username: nova password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /nova_api @@ -1674,7 +1674,7 @@ endpoints: username: nova password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /nova_cell0 diff --git a/helm-configs/octavia/octavia-helm-overrides.yaml b/helm-configs/octavia/octavia-helm-overrides.yaml index 1a30a9e2..2865d4c9 100644 --- a/helm-configs/octavia/octavia-helm-overrides.yaml +++ b/helm-configs/octavia/octavia-helm-overrides.yaml @@ -466,7 +466,7 @@ endpoints: username: octavia password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /octavia diff --git a/helm-configs/placement/placement-helm-overrides.yaml b/helm-configs/placement/placement-helm-overrides.yaml index 9d85dd6e..f6a2cc8c 100644 --- a/helm-configs/placement/placement-helm-overrides.yaml +++ b/helm-configs/placement/placement-helm-overrides.yaml @@ -206,7 +206,7 @@ endpoints: username: nova password: password hosts: - default: mariadb-galera-primary + default: maxscale-galera host_fqdn_override: default: null path: /placement diff --git a/kustomize/grafana/base/grafana-database.yaml b/kustomize/grafana/base/grafana-database.yaml index 1accfcbf..4845db21 100644 --- a/kustomize/grafana/base/grafana-database.yaml +++ b/kustomize/grafana/base/grafana-database.yaml @@ -1,4 +1,4 @@ -apiVersion: mariadb.mmontes.io/v1alpha1 +apiVersion: k8s.mariadb.com/v1alpha1 kind: Database metadata: name: grafana @@ -13,7 +13,7 @@ spec: requeueInterval: 30s retryInterval: 5s --- -apiVersion: mariadb.mmontes.io/v1alpha1 +apiVersion: k8s.mariadb.com/v1alpha1 kind: User metadata: name: grafana @@ -32,7 +32,7 @@ spec: requeueInterval: 30s retryInterval: 5s --- -apiVersion: mariadb.mmontes.io/v1alpha1 +apiVersion: k8s.mariadb.com/v1alpha1 kind: Grant metadata: name: grant diff --git a/kustomize/mariadb-cluster/aio/kustomization.yaml b/kustomize/mariadb-cluster/aio/kustomization.yaml index 5b0a66d9..7e5e7e83 100644 --- a/kustomize/mariadb-cluster/aio/kustomization.yaml +++ b/kustomize/mariadb-cluster/aio/kustomization.yaml @@ -12,3 +12,10 @@ patches: - op: replace path: /spec/galera/enabled value: false + - target: + kind: MaxScale + name: maxscale-galera + patch: |- + - op: replace + path: /spec/replicas + value: 1 diff --git a/kustomize/mariadb-cluster/base/kustomization.yaml b/kustomize/mariadb-cluster/base/kustomization.yaml index f297b151..c074ce60 100644 --- a/kustomize/mariadb-cluster/base/kustomization.yaml +++ b/kustomize/mariadb-cluster/base/kustomization.yaml @@ -1,4 +1,5 @@ resources: - mariadb-configmap.yaml + - mariadb-maxscale.yaml - mariadb-galera.yaml - mariadb-backup.yaml diff --git a/kustomize/mariadb-cluster/base/mariadb-galera.yaml b/kustomize/mariadb-cluster/base/mariadb-galera.yaml index 6fc77a9e..aa737aa3 100644 --- a/kustomize/mariadb-cluster/base/mariadb-galera.yaml +++ b/kustomize/mariadb-cluster/base/mariadb-galera.yaml @@ -28,21 +28,8 @@ spec: runAsUser: 0 # point to an existing MaxScale instance. Doing this will delegate tasks such as primary failover to MaxScale. - # maxScaleRef: - # name: maxscale - - # provision a MaxScale instance and set 'spec.maxScaleRef' automatically. - maxScale: - enabled: false - - kubernetesService: - type: LoadBalancer - annotations: - metallb.universe.tf/address-pool: primary - - connection: - secretName: mxs-galera-conn - port: 3306 + maxScaleRef: + name: maxscale-galera galera: enabled: true @@ -132,8 +119,6 @@ spec: resources: requests: memory: 256Mi - limits: - memory: 16Gi metrics: enabled: true diff --git a/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml b/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml new file mode 100644 index 00000000..350d8ca0 --- /dev/null +++ b/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml @@ -0,0 +1,132 @@ +apiVersion: k8s.mariadb.com/v1alpha1 +kind: MaxScale +metadata: + name: maxscale-galera +spec: + replicas: 3 + + mariaDbRef: + name: mariadb-galera + namespace: openstack + + services: + - name: rw-router + router: readwritesplit + params: + transaction_replay: "true" + transaction_replay_attempts: "10" + transaction_replay_timeout: "5s" + max_slave_connections: "255" + max_replication_lag: "3s" + master_accept_reads: "true" + listener: + name: rw-listener + port: 3306 + protocol: MariaDBProtocol + params: + connection_metadata: "tx_isolation=auto" + suspend: false + suspend: false + - name: rconn-master-router + router: readconnroute + params: + router_options: "master" + max_replication_lag: "3s" + master_accept_reads: "true" + listener: + port: 3307 + - name: rconn-slave-router + router: readconnroute + params: + router_options: "slave" + max_replication_lag: "3s" + listener: + port: 3308 + + monitor: + name: mariadb-monitor + module: galeramon + interval: 2s + cooperativeMonitoring: majority_of_all + params: + disable_master_failback: "false" + available_when_donor: "false" + disable_master_role_setting: "false" + suspend: false + + admin: + port: 8989 + guiEnabled: true + + config: + params: + log_info: "true" + volumeClaimTemplate: + resources: + requests: + storage: 100Mi + accessModes: + - ReadWriteOnce + storageClassName: general + sync: + database: mysql + interval: 5s + timeout: 10s + + auth: + generate: true + adminUsername: mariadb-operator + adminPasswordSecretKeyRef: + name: maxscale + key: password + deleteDefaultAdmin: true + clientUsername: maxscale-galera-client + clientPasswordSecretKeyRef: + name: maxscale + key: password + clientMaxConnections: 1024 + serverUsername: maxscale-galera-server + serverPasswordSecretKeyRef: + name: maxscale + key: password + serverMaxConnections: 1024 + monitorUsername: maxscale-galera-monitor + monitorPasswordSecretKeyRef: + name: maxscale + key: password + monitorMaxConnections: 128 + syncUsername: maxscale-galera-sync + syncPasswordSecretKeyRef: + name: maxscale + key: password + syncMaxConnections: 128 + + securityContext: + allowPrivilegeEscalation: false + + updateStrategy: + type: RollingUpdate + + kubernetesService: + type: LoadBalancer + annotations: + metallb.universe.tf/address-pool: primary + + connection: + secretName: mxs-galera-conn + port: 3306 + + resources: + requests: + memory: 128Mi + + affinity: + enableAntiAffinity: true + + tolerations: + - key: "k8s.mariadb.com/ha" + operator: "Exists" + effect: "NoSchedule" + + podDisruptionBudget: + maxUnavailable: 33% diff --git a/kustomize/octavia/base/octavia-agent.yaml b/kustomize/octavia/base/octavia-agent.yaml index 60fc81d8..58fb12ad 100644 --- a/kustomize/octavia/base/octavia-agent.yaml +++ b/kustomize/octavia/base/octavia-agent.yaml @@ -81,7 +81,7 @@ spec: - name: PATH value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ - name: DEPENDENCY_SERVICE - value: "openstack:mariadb-galera-primary,openstack:keystone-api,openstack:rabbitmq-nodes,openstack:memcached,openstack:neutron-server" + value: "openstack:maxscale-galera,openstack:keystone-api,openstack:rabbitmq-nodes,openstack:memcached,openstack:neutron-server" - name: DEPENDENCY_JOBS value: "octavia-db-sync,octavia-ks-user,octavia-ks-endpoints" - name: DEPENDENCY_DAEMONSET @@ -187,4 +187,4 @@ spec: secretName: octavia-etc defaultMode: 0444 - emptyDir: {} - name: pod-run-octavia \ No newline at end of file + name: pod-run-octavia