microcluster/internal/db: Add the API extensions check in the `db.Ope…

…n` function

Here, we are checking for set equality between this node's current extensions and the other node's.
In some cases, when this node's extensions set is 'bigger', we allow the other nodes to be upgraded
in the hope to make all the system's extensions align properly. If the extensions sets are equal, we continue the checking process
as usual.

There are some cases that are not recoverable like the current node lacking extensions compared to the other one
or the case where two nodes have a partial or no extensions overlap. In these case, we indicate what are the
lacking extensions in the error messages.

Signed-off-by: Gabriel Mougard <[email protected]>

internal/db/db.go

@@ -40,36 +40,59 @@ func (db *DB) Open(ext *extensions.Extensions, bootstrap bool, project string) e
 	if !bootstrap {
 		checkVersions := func(ctx context.Context, current int, tx *sql.Tx) error {
 			schemaVersion := newSchema.Version()
-			err = cluster.UpdateClusterMemberSchemaVersion(tx, schemaVersion, db.listenAddr.URL.Host)
+			info := cluster.InternalClusterMemberVersioningInfo{
+				SchemaVersion: schemaVersion,
+				Extensions:    ext,
+			}
+
+			err = cluster.UpdateClusterMemberSchemaVersionAndAPIExtensions(tx, info, db.listenAddr.URL.Host)
 			if err != nil {
 				return fmt.Errorf("Failed to update schema version when joining cluster: %w", err)
 			}
 
-			versions, err := cluster.GetClusterMemberSchemaVersions(ctx, tx)
+			versioningInfo, err := cluster.GetClusterMemberSchemaVersionsAndAPIExtensions(ctx, tx)
 			if err != nil {
 				return fmt.Errorf("Failed to get other members' schema versions: %w", err)
 			}
 
-			for _, version := range versions {
-				if schemaVersion == version {
-					// Versions are equal, there's hope for the
-					// update. Let's check the next node.
-					continue
+			for _, versionInfo := range versioningInfo {
+				isSameVersion := ext.IsSameVersion(versionInfo.Extensions) == nil
+				if isSameVersion {
+					// All the API extensions are equal, now check the schema version
+					if schemaVersion == versionInfo.SchemaVersion {
+						// Versions and API extensions are equal, there's hope for the
+						// update. Let's check the next node.
+						continue
+					}
+
+					if schemaVersion > versionInfo.SchemaVersion {
+						// Our version is bigger, we should stop here
+						// and wait for other nodes to be upgraded and
+						// restarted.
+						otherNodesBehind = true
+						return schema.ErrGracefulAbort
+					}
+
+					// Another node has a version greater than ours
+					// and presumably is waiting for other nodes
+					// to upgrade. Let's error out and shutdown
+					// since we need a greater version.
+					return fmt.Errorf("Cluster expects Schema version (%d) but this system has (%d), please upgrade", versionInfo.SchemaVersion, schemaVersion)
 				}
 
-				if schemaVersion > version {
-					// Our version is bigger, we should stop here
-					// and wait for other nodes to be upgraded and
-					// restarted.
-					otherNodesBehind = true
-					return schema.ErrGracefulAbort
+				if ext.Version() <= versionInfo.Extensions.Version() {
+					// Another node has an API extension set bigger than ours
+					// and presumably is waiting for other nodes
+					// to upgrade. Let's error out and shutdown
+					// since we need a greater version.
+					return fmt.Errorf("Cluster expects API version (%d) but this system has (%d), please upgrade", versionInfo.Extensions.Version(), ext.Version())
 				}
 
-				// Another node has a version greater than ours
-				// and presumeably is waiting for other nodes
-				// to upgrade. Let's error out and shutdown
-				// since we need a greater version.
-				return fmt.Errorf("this node's version is behind, please upgrade")
+				// Our version is bigger, we should stop here
+				// and wait for other nodes to be upgraded and
+				// restarted.
+				otherNodesBehind = true
+				return schema.ErrGracefulAbort
 			}
 
 			return nil