[DPE-8050] Backup action cluster checks (#679)

Author: sinclert-canonical

actions.yaml

@@ -45,6 +45,13 @@ set-tls-private-key:
 create-backup:
   description: Create a database backup using xtrabackup.
     S3 credentials are retrieved from a relation with the S3 integrator charm.
+  params:
+    force:
+      type: boolean
+      default: False
+      description: |
+        Whether to ignore cluster health concerns and create the backup regardless.
+        Use it with caution, as it can potentially create a backup from stale data.
 
 list-backups:
   description: List available backup_ids in the S3 bucket and path provided by the S3 integrator charm.

docs/how-to/back-up-and-restore/create-a-backup.md

@@ -15,9 +15,18 @@ Once `juju status` shows Charmed MySQL as `active` and `idle`, you can create yo
 juju run mysql/leader create-backup
 ```
 
-If you have a cluster of one unit, you can run the `create-backup` action on `mysql-k8s/leader` (which will also be the primary unit). 
+If you have a cluster of one unit, you can run the `create-backup` action on the leader (which will also be the primary unit).
+Otherwise, you must run the `create-backup` action on a non-primary unit. To find the primary, see `juju status` or
+run `juju run mysql/leader get-cluster-status` to find the primary unit.
 
-Otherwise, you must run the `create-backup` action on a non-primary unit. To find the primary, see `juju status` or run `juju run mysql-k8s/leader get-cluster-status` to find the primary unit).
+The `create-backup` action validates that the unit in charge of the backup is healthy, by:
+- Checking that the MySQL cluster is in a valid state (`OK` or `OK_PARTIAL` from the InnoDB [cluster status](https://dev.mysql.com/doc/mysql-shell/8.0/en/monitoring-innodb-cluster.html))
+- Checking that the MySQL instance is in a valid state (`ONLINE` from Replication [member states](https://dev.mysql.com/doc/refman/8.0/en/group-replication-server-states.html).
+
+In order to override these precautions, use the `force` flag:
+```shell
+juju run mysql/leader create-backup force=True
+```
 
 ## List backups
 You can list your available, failed, and in progress backups by running the `list-backups` command:

lib/charms/mysql/v0/backups.py

@@ -57,6 +57,7 @@ def is_unit_blocked(self) -> bool:
     S3Requirer,
 )
 from charms.mysql.v0.mysql import (
+    MySQLClusterState,
     MySQLConfigureInstanceError,
     MySQLCreateClusterError,
     MySQLCreateClusterSetError,
@@ -66,6 +67,7 @@ def is_unit_blocked(self) -> bool:
     MySQLExecuteBackupCommandsError,
     MySQLInitializeJujuOperationsTableError,
     MySQLKillSessionError,
+    MySQLMemberState,
     MySQLNoMemberStateError,
     MySQLOfflineModeAndHiddenInstanceExistsError,
     MySQLPrepareBackupForRestoreError,
@@ -111,7 +113,7 @@ def is_unit_blocked(self) -> bool:
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 16
+LIBPATCH = 17
 
 ANOTHER_S3_CLUSTER_REPOSITORY_ERROR_MESSAGE = "S3 repository claimed by another cluster"
 MOVE_RESTORED_CLUSTER_TO_ANOTHER_S3_REPOSITORY_ERROR = (
@@ -280,6 +282,7 @@ def _pre_create_backup_checks(self, event: ActionEvent) -> bool:
     def _on_create_backup(self, event: ActionEvent) -> None:
         """Handle the create backup action."""
         logger.info("A backup has been requested on unit")
+        force = event.params.get("force", False)
 
         if not self._pre_create_backup_checks(event):
             return
@@ -295,9 +298,16 @@ def _on_create_backup(self, event: ActionEvent) -> None:
 
         backup_path = str(pathlib.Path(s3_parameters["path"]) / datetime_backup_requested)
 
+        # Check if this cluster can perform backup
+        can_cluster_perform_backup, validation_message = self._can_cluster_perform_backup()
+        if not (can_cluster_perform_backup or force):
+            logger.error(f"Backup failed: {validation_message}")
+            event.fail(validation_message or "")
+            return
+
         # Check if this unit can perform backup
         can_unit_perform_backup, validation_message = self._can_unit_perform_backup()
-        if not can_unit_perform_backup:
+        if not (can_unit_perform_backup or force):
             logger.error(f"Backup failed: {validation_message}")
             event.fail(validation_message or "")
             return
@@ -355,6 +365,21 @@ def _on_create_backup(self, event: ActionEvent) -> None:
         })
         self.charm._on_update_status(None)
 
+    def _can_cluster_perform_backup(self) -> tuple[bool, str | None]:
+        """Validates whether this cluster can perform a backup.
+
+        Returns: tuple of (success, error_message)
+        """
+        cluster_status = self.charm._mysql.get_cluster_status()
+        if not cluster_status:
+            return False, "Cluster status unknown"
+
+        cluster_status = cluster_status["defaultreplicaset"]["status"]
+        if cluster_status not in [MySQLClusterState.OK, MySQLClusterState.OK_PARTIAL]:
+            return False, "Cluster is not in a healthy state"
+
+        return True, None
+
     def _can_unit_perform_backup(self) -> tuple[bool, str | None]:
         """Validates whether this unit can perform a backup.
 
@@ -381,7 +406,7 @@ def _can_unit_perform_backup(self) -> tuple[bool, str | None]:
         if role == "primary" and self.charm.app.planned_units() > 1:
             return False, "Unit cannot perform backups as it is the cluster primary"
 
-        if state in ["recovering", "offline", "error"]:
+        if state not in [MySQLMemberState.ONLINE]:
             return False, f"Unit cannot perform backups as its state is {state}"
 
         return True, None

lib/charms/mysql/v0/mysql.py

@@ -127,7 +127,7 @@ def wait_until_mysql_connection(self) -> None:
 # Increment this major API version when introducing breaking changes
 LIBAPI = 0
 
-LIBPATCH = 93
+LIBPATCH = 94
 
 UNIT_TEARDOWN_LOCKNAME = "unit-teardown"
 UNIT_ADD_LOCKNAME = "unit-add"
@@ -916,11 +916,7 @@ def get_cluster_endpoints(self, relation_name: str) -> tuple[str, str, str]:
 
         return ",".join(rw_endpoints), ",".join(ro_endpoints), ",".join(no_endpoints)
 
-    def get_secret(
-        self,
-        scope: Scopes,
-        key: str,
-    ) -> str | None:
+    def get_secret(self, scope: Scopes, key: str) -> str | None:
         """Get secret from the secret storage.
 
         Retrieve secret from juju secrets backend if secret exists there.
@@ -1012,7 +1008,18 @@ class MySQLMemberState(str, enum.Enum):
 class MySQLClusterState(str, enum.Enum):
     """MySQL Cluster state."""
 
+    # TODO: python 3.11 has new enum.StrEnum
+    #       that can remove str inheritance
+
     OK = "ok"
+    OK_PARTIAL = "ok_partial"
+    OK_NO_TOLERANCE = "ok_no_tolerance"
+    OK_NO_TOLERANCE_PARTIAL = "ok_no_tolerance_partial"
+    NO_QUORUM = "no_quorum"
+    OFFLINE = "offline"
+    ERROR = "error"
+    UNREACHABLE = "unreachable"
+    UNKNOWN = "unknown"
     FENCED = "fenced_writes"