From 2daf725e69ded052bc89bed3b195cd95ab39ed10 Mon Sep 17 00:00:00 2001 From: Anisur Rahman Date: Fri, 30 Aug 2024 12:48:13 +0600 Subject: [PATCH 01/10] Add logical backup/restore Add auto backup Signed-off-by: Anisur Rahman --- docs/guides/mysql/README.md | 2 +- docs/guides/mysql/backup/kubestash/_index.md | 10 + .../examples/backupconfiguration.yaml | 37 + .../examples/backupstorage.yaml | 17 + .../examples/restoresession.yaml | 21 + .../examples/retentionpolicy.yaml | 15 + .../examples/sample-mysql.yaml | 18 + .../kubestash/application-level/index.md | 761 ++++++++++++++++++ .../auto-backup/examples/backupstorage.yaml | 17 + .../examples/customize-backupblueprint.yaml | 41 + .../examples/default-backupblueprint.yaml | 37 + .../auto-backup/examples/retentionpolicy.yaml | 15 + .../auto-backup/examples/sample-mysql-2.yaml | 25 + .../auto-backup/examples/sample-mysql.yaml | 20 + .../backup/kubestash/auto-backup/index.md | 738 +++++++++++++++++ .../examples/backup/multiple-backends.yaml | 49 ++ .../examples/backup/passing-args.yaml | 38 + .../examples/backup/passing-databases.yaml | 38 + .../examples/backup/resources-limit.yaml | 46 ++ .../examples/backup/specific-user.yaml | 41 + .../examples/common/gcs-backupstorage.yaml | 17 + .../examples/common/retentionpolicy.yaml | 15 + .../examples/common/s3-backupstorage.yaml | 19 + .../examples/common/sample-mysql.yaml | 18 + .../examples/restore/passing-args.yaml | 23 + .../examples/restore/resources-limit.yaml | 30 + .../examples/restore/specific-snapshot.yaml | 21 + .../examples/restore/specific-user.yaml | 26 + .../backup/kubestash/customization/index.md | 444 ++++++++++ .../logical/examples/backupconfiguration.yaml | 36 + .../logical/examples/backupstorage.yaml | 17 + .../logical/examples/restored-mysql.yaml | 20 + .../logical/examples/restoresession.yaml | 21 + .../logical/examples/retentionpolicy.yaml | 15 + .../logical/examples/sample-mysql.yaml | 18 + .../mysql/backup/kubestash/logical/index.md | 756 +++++++++++++++++ .../overview/images/backup_overview.svg | 1 + .../overview/images/kubedb_plus_kubestash.svg | 21 + .../overview/images/restore_overview.svg | 1 + .../mysql/backup/kubestash/overview/index.md | 98 +++ docs/guides/mysql/backup/stash/_index.md | 10 + .../auto-backup/examples/backupblueprint.yaml | 0 .../auto-backup/examples/sample-mysql-2.yaml | 0 .../auto-backup/examples/sample-mysql-3.yaml | 0 .../auto-backup/examples/sample-mysql.yaml | 0 .../auto-backup/images/sample-mysql-2.png | Bin .../auto-backup/images/sample-mysql-3.png | Bin .../auto-backup/images/sample-mysql.png | Bin .../backup/{ => stash}/auto-backup/index.md | 22 +- .../backup/multi-retention-policy.yaml | 0 .../examples/backup/passing-args.yaml | 0 .../examples/backup/resource-limit.yaml | 0 .../examples/backup/specific-user.yaml | 0 .../customization/examples/repository.yaml | 0 .../examples/restore/passing-args.yaml | 0 .../examples/restore/resource-limit.yaml | 0 .../examples/restore/specific-snapshot.yaml | 0 .../examples/restore/specific-user.yaml | 0 .../customization/examples/sample-mysql.yaml | 0 .../backup/{ => stash}/customization/index.md | 4 +- .../overview/images/backup_overview.svg | 0 .../overview/images/restore_overview.svg | 0 .../backup/{ => stash}/overview/index.md | 10 +- .../examples/backupconfiguration.yaml | 0 .../standalone/examples/repository.yaml | 0 .../standalone/examples/restored-mysql.yaml | 0 .../standalone/examples/restoresession.yaml | 0 .../standalone/examples/sample-mysql.yaml | 0 .../standalone/images/sample-mysql-backup.png | Bin .../backup/{ => stash}/standalone/index.md | 18 +- docs/guides/mysql/pitr/archiver.md | 2 +- 71 files changed, 3640 insertions(+), 29 deletions(-) create mode 100644 docs/guides/mysql/backup/kubestash/_index.md create mode 100644 docs/guides/mysql/backup/kubestash/application-level/examples/backupconfiguration.yaml create mode 100644 docs/guides/mysql/backup/kubestash/application-level/examples/backupstorage.yaml create mode 100644 docs/guides/mysql/backup/kubestash/application-level/examples/restoresession.yaml create mode 100644 docs/guides/mysql/backup/kubestash/application-level/examples/retentionpolicy.yaml create mode 100644 docs/guides/mysql/backup/kubestash/application-level/examples/sample-mysql.yaml create mode 100644 docs/guides/mysql/backup/kubestash/application-level/index.md create mode 100644 docs/guides/mysql/backup/kubestash/auto-backup/examples/backupstorage.yaml create mode 100644 docs/guides/mysql/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml create mode 100644 docs/guides/mysql/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml create mode 100644 docs/guides/mysql/backup/kubestash/auto-backup/examples/retentionpolicy.yaml create mode 100644 docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql-2.yaml create mode 100644 docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql.yaml create mode 100644 docs/guides/mysql/backup/kubestash/auto-backup/index.md create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/backup/multiple-backends.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/backup/passing-args.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/backup/passing-databases.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/backup/resources-limit.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/backup/specific-user.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/common/gcs-backupstorage.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/common/retentionpolicy.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/common/s3-backupstorage.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/common/sample-mysql.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/restore/passing-args.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/restore/resources-limit.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/restore/specific-snapshot.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/examples/restore/specific-user.yaml create mode 100644 docs/guides/mysql/backup/kubestash/customization/index.md create mode 100644 docs/guides/mysql/backup/kubestash/logical/examples/backupconfiguration.yaml create mode 100644 docs/guides/mysql/backup/kubestash/logical/examples/backupstorage.yaml create mode 100644 docs/guides/mysql/backup/kubestash/logical/examples/restored-mysql.yaml create mode 100644 docs/guides/mysql/backup/kubestash/logical/examples/restoresession.yaml create mode 100644 docs/guides/mysql/backup/kubestash/logical/examples/retentionpolicy.yaml create mode 100644 docs/guides/mysql/backup/kubestash/logical/examples/sample-mysql.yaml create mode 100644 docs/guides/mysql/backup/kubestash/logical/index.md create mode 100644 docs/guides/mysql/backup/kubestash/overview/images/backup_overview.svg create mode 100644 docs/guides/mysql/backup/kubestash/overview/images/kubedb_plus_kubestash.svg create mode 100644 docs/guides/mysql/backup/kubestash/overview/images/restore_overview.svg create mode 100644 docs/guides/mysql/backup/kubestash/overview/index.md create mode 100644 docs/guides/mysql/backup/stash/_index.md rename docs/guides/mysql/backup/{ => stash}/auto-backup/examples/backupblueprint.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/auto-backup/examples/sample-mysql-2.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/auto-backup/examples/sample-mysql-3.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/auto-backup/examples/sample-mysql.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/auto-backup/images/sample-mysql-2.png (100%) rename docs/guides/mysql/backup/{ => stash}/auto-backup/images/sample-mysql-3.png (100%) rename docs/guides/mysql/backup/{ => stash}/auto-backup/images/sample-mysql.png (100%) rename docs/guides/mysql/backup/{ => stash}/auto-backup/index.md (97%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/backup/multi-retention-policy.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/backup/passing-args.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/backup/resource-limit.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/backup/specific-user.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/repository.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/restore/passing-args.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/restore/resource-limit.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/restore/specific-snapshot.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/restore/specific-user.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/examples/sample-mysql.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/customization/index.md (98%) rename docs/guides/mysql/backup/{ => stash}/overview/images/backup_overview.svg (100%) rename docs/guides/mysql/backup/{ => stash}/overview/images/restore_overview.svg (100%) rename docs/guides/mysql/backup/{ => stash}/overview/index.md (94%) rename docs/guides/mysql/backup/{ => stash}/standalone/examples/backupconfiguration.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/standalone/examples/repository.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/standalone/examples/restored-mysql.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/standalone/examples/restoresession.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/standalone/examples/sample-mysql.yaml (100%) rename docs/guides/mysql/backup/{ => stash}/standalone/images/sample-mysql-backup.png (100%) rename docs/guides/mysql/backup/{ => stash}/standalone/index.md (97%) diff --git a/docs/guides/mysql/README.md b/docs/guides/mysql/README.md index 8e3ce1fa64..ad11191d13 100644 --- a/docs/guides/mysql/README.md +++ b/docs/guides/mysql/README.md @@ -45,7 +45,7 @@ aliases: ## User Guide - [Quickstart MySQL](/docs/guides/mysql/quickstart/index.md) with KubeDB Operator. -- [Backup & Restore](/docs/guides/mysql/backup/overview/index.md) MySQL databases using Stash. +- [Backup & Restore](/docs/guides/mysql/backup/stash/overview/index.md) MySQL databases using Stash. - Initialize [MySQL with Script](/docs/guides/mysql/initialization/index.md). - Monitor your MySQL database with KubeDB using [out-of-the-box Prometheus operator](/docs/guides/mysql/monitoring/prometheus-operator/index.md). - Monitor your MySQL database with KubeDB using [out-of-the-box builtin-Prometheus](/docs/guides/mysql/monitoring/builtin-prometheus/index.md). diff --git a/docs/guides/mysql/backup/kubestash/_index.md b/docs/guides/mysql/backup/kubestash/_index.md new file mode 100644 index 0000000000..a5d2844a63 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/_index.md @@ -0,0 +1,10 @@ +--- +title: Backup & Restore MySQL | KubeStash +menu: + docs_{{ .version }}: + identifier: guides-mysql-backup-stashv2 + name: KubeStash (aka Stash 2.0) + parent: guides-mysql-backup + weight: 50 +menu_name: docs_{{ .version }} +--- \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/application-level/examples/backupconfiguration.yaml b/docs/guides/mysql/backup/kubestash/application-level/examples/backupconfiguration.yaml new file mode 100644 index 0000000000..4d9e1fbae3 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/application-level/examples/backupconfiguration.yaml @@ -0,0 +1,37 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: manifest-backup + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/application-level/examples/backupstorage.yaml b/docs/guides/mysql/backup/kubestash/application-level/examples/backupstorage.yaml new file mode 100644 index 0000000000..6ab3df02ac --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/application-level/examples/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/application-level/examples/restoresession.yaml b/docs/guides/mysql/backup/kubestash/application-level/examples/restoresession.yaml new file mode 100644 index 0000000000..fa1fe6e950 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/application-level/examples/restoresession.yaml @@ -0,0 +1,21 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + manifestOptions: + restoreNamespace: dev + mySQL: + db: true + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore + - name: manifest-restore \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/application-level/examples/retentionpolicy.yaml b/docs/guides/mysql/backup/kubestash/application-level/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/application-level/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/application-level/examples/sample-mysql.yaml b/docs/guides/mysql/backup/kubestash/application-level/examples/sample-mysql.yaml new file mode 100644 index 0000000000..89aa071feb --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/application-level/examples/sample-mysql.yaml @@ -0,0 +1,18 @@ +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: sample-mysql + namespace: demo +spec: + version: "8.2.0" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/application-level/index.md b/docs/guides/mysql/backup/kubestash/application-level/index.md new file mode 100644 index 0000000000..805d84903d --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/application-level/index.md @@ -0,0 +1,761 @@ +--- +title: Application Level Backup & Restore MySQL | KubeStash +description: Application Level Backup and Restore using KubeStash +menu: + docs_{{ .version }}: + identifier: guides-application-level-backup-stashv2 + name: Application Level Backup + parent: guides-mysql-backup-stashv2 + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Application Level Backup and Restore MySQL database using KubeStash + +KubeStash offers application-level backup and restore functionality for `MySQL` databases. It captures both manifest and logical data backups of any `MySQL` database in a single snapshot. During the restore process, KubeStash first applies the `MySQL` manifest to the cluster and then restores the data into it. + +This guide will give you how you can take application-level backup and restore your `MySQL` databases using `Kubestash`. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. +- Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). +- Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). +- Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- If you are not familiar with how KubeStash backup and restore MySQL databases, please check the following guide [here](/docs/guides/mysql/backup/kubestash/overview/index.md). + +You should be familiar with the following `KubeStash` concepts: + +- [BackupStorage](https://kubestash.com/docs/latest/concepts/crds/backupstorage/) +- [BackupConfiguration](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/) +- [BackupSession](https://kubestash.com/docs/latest/concepts/crds/backupsession/) +- [RestoreSession](https://kubestash.com/docs/latest/concepts/crds/restoresession/) +- [Addon](https://kubestash.com/docs/latest/concepts/crds/addon/) +- [Function](https://kubestash.com/docs/latest/concepts/crds/function/) +- [Task](https://kubestash.com/docs/latest/concepts/crds/addon/#task-specification) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/guides/mysql/backup/kubestash/application-level/examples](docs/guides/mysql/backup/kubestash/application-level/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + +## Backup MySQL + +KubeStash supports backups for `MySQL` instances across different configurations, including Standalone, Group Replication, and InnoDB Cluster setups. In this demonstration, we'll focus on a `MySQL` database using Group Replication. The backup and restore process is similar for Standalone and InnoDB Cluster configurations as well. + +This section will demonstrate how to take application-level backup of a `MySQL` database. Here, we are going to deploy a `MySQL` database using KubeDB. Then, we are going to back up the database at the application level to a `GCS` bucket. Finally, we will restore the entire `MySQL` database. + +### Deploy Sample MySQL Database + +Let's deploy a sample `MySQL` database and insert some data into it. + +**Create MySQL CR:** + +Below is the YAML of a sample `MySQL` CR that we are going to create for this tutorial: + +```yaml +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: sample-mysql + namespace: demo +spec: + version: "8.2.0" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + deletionPolicy: WipeOut +``` + +Here, +- `.spec.topology` specifies about the clustering configuration of MySQL. +- `.Spec.topology.mode` specifies the mode of MySQL Cluster. During the demonstration we consider to use `GroupReplication`. + +Create the above `MySQL` CR, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/sample-mysql.yaml +mysql.kubedb.com/sample-mysql created +``` + +KubeDB will deploy a MySQL database according to the above specification. It will also create the necessary Secrets and Services to access the database. + +Let's check if the database is ready to use, + +```bash +$ kubectl get mysqls.kubedb.com -n demo +NAME VERSION STATUS AGE +sample-mysql 8.2.0 Ready 4m22s +``` + +The database is `Ready`. Verify that KubeDB has created a `Secret` and a `Service` for this database using the following commands, + +```bash +$ kubectl get secret -n demo +NAME TYPE DATA AGE +sample-mysql-auth Opaque 2 4m58s + +$ kubectl get service -n demo -l=app.kubernetes.io/instance=sample-mysql +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +sample-mysql ClusterIP 10.96.55.61 3306/TCP 97s +sample-mysql-pods ClusterIP None 3306/TCP 97s +sample-mysql-standby ClusterIP 10.96.211.186 3306/TCP 97 +``` + +Here, we have to use service `sample-mysql` and secret `sample-mysql-auth` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/mysql/concepts/appbinding/index.md) CR that holds the necessary information to connect with the database. + +**Verify AppBinding:** + +Verify that the `AppBinding` has been created successfully using the following command, + +```bash +$ kubectl get appbindings -n demo +NAME AGE +sample-mysql 9m24s +``` + +Let's check the YAML of the above `AppBinding`, + +```bash +$ kubectl get appbindings -n demo sample-mysql -o yaml +``` + +```yaml +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: sample-mysql + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: mysqls.kubedb.com + name: sample-mysql + namespace: demo + ownerReferences: + - apiVersion: kubedb.com/v1 + blockOwnerDeletion: true + controller: true + kind: MySQL + name: sample-mysql + uid: edde3e8b-7775-4f91-85a9-4ba4b96315f7 + resourceVersion: "5126" + uid: 86c9a149-f8ab-44c4-947f-5f9b402aad6c +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo + clientConfig: + service: + name: sample-mysql + path: / + port: 3306 + scheme: tcp + url: tcp(sample-mysql.demo.svc:3306)/ + ... + ... + secret: + name: sample-mysql-auth + type: kubedb.com/mysql + version: 8.2.0 +``` + +KubeStash uses the `AppBinding` CR to connect with the target database. It requires the following two fields to set in AppBinding's `.spec` section. + +- `.spec.clientConfig.service.name` specifies the name of the Service that connects to the database. +- `.spec.secret` specifies the name of the Secret that holds necessary credentials to access the database. +- `spec.type` specifies the types of the app that this AppBinding is pointing to. KubeDB generated AppBinding follows the following format: `/`. + +**Insert Sample Data:** + +Now, we are going to exec into the database pod and create some sample data. At first, find out the database Pod using the following command, + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-mysql" +NAME READY STATUS RESTARTS AGE +sample-mysql-0 2/2 Running 0 33m +sample-mysql-1 2/2 Running 0 33m +sample-mysql-2 2/2 Running 0 33m +``` + +And copy the username and password of the `root` user to access into `mysql` shell. + +```bash +$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.username}'| base64 -d +root⏎ + +$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.password}'| base64 -d +DZfmUZd14fNEEOU4⏎ +``` + +Now, Lets exec into the Pod to enter into `mysql` shell and create a database and a table, + +```bash +$ kubectl exec -it -n demo sample-mysql-0 -- mysql --user=root --password=DZfmUZd14fNEEOU4 +Defaulted container "mysql" out of: mysql, mysql-init (init) +mysql: [Warning] Using a password on the command line interface can be insecure. +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 977 +Server version: 8.2.0 MySQL Community Server - GPL + +Copyright (c) 2000, 2023, Oracle and/or its affiliates. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> CREATE DATABASE playground; +Query OK, 1 row affected (0.01 sec) + +mysql> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| information_schema | +| mysql | +| performance_schema | +| playground | +| sys | ++--------------------+ +5 rows in set (0.00 sec) + +mysql> CREATE TABLE playground.equipment ( id INT NOT NULL AUTO_INCREMENT, type VARCHAR(50), quant INT, color VARCHAR(25), PRIMARY KEY(id)); +Query OK, 0 rows affected (0.01 sec) + +mysql> SHOW TABLES IN playground; ++----------------------+ +| Tables_in_playground | ++----------------------+ +| equipment | ++----------------------+ +1 row in set (0.01 sec) + +mysql> INSERT INTO playground.equipment (type, quant, color) VALUES ("slide", 2, "blue"); +Query OK, 1 row affected (0.01 sec) + +mysql> SELECT * FROM playground.equipment; ++----+-------+-------+-------+ +| id | type | quant | color | ++----+-------+-------+-------+ +| 1 | slide | 2 | blue | ++----+-------+-------+-------+ +1 row in set (0.00 sec) + +mysql> exit +Bye +``` +Now, we are ready to backup the database. + +### Prepare Backend + +We are going to store our backed up data into a GCS bucket. We have to create a Secret with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). + +**Create Secret:** + +Let's create a secret called `gcs-secret` with access credentials to our desired GCS bucket, + +```bash +$ echo -n '' > GOOGLE_PROJECT_ID +$ cat /path/to/downloaded-sa-key.json > GOOGLE_SERVICE_ACCOUNT_JSON_KEY +$ kubectl create secret generic -n demo gcs-secret \ + --from-file=./GOOGLE_PROJECT_ID \ + --from-file=./GOOGLE_SERVICE_ACCOUNT_JSON_KEY +secret/gcs-secret created +``` + +**Create BackupStorage:** + +Now, create a `BackupStorage` using this secret. Below is the YAML of `BackupStorage` CR we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete +``` + +Let's create the BackupStorage we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/backupstorage.yaml +backupstorage.storage.kubestash.com/gcs-storage created +``` + +Now, we are ready to backup our database to our desired backend. + +**Create RetentionPolicy:** + +Now, let's create a `RetentionPolicy` to specify how the old Snapshots should be cleaned up. + +Below is the YAML of the `RetentionPolicy` object that we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Let’s create the above `RetentionPolicy`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/retentionpolicy.yaml +retentionpolicy.storage.kubestash.com/demo-retention created +``` + +### Backup + +We have to create a `BackupConfiguration` targeting respective `sample-mysql` MySQL database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. + +At first, we need to create a secret with a Restic password for backup data encryption. + +**Create Secret:** + +Let's create a secret called `encrypt-secret` with the Restic password, + +```bash +$ echo -n 'changeit' > RESTIC_PASSWORD +$ kubectl create secret generic -n demo encrypt-secret \ + --from-file=./RESTIC_PASSWORD \ +secret "encrypt-secret" created +``` + +**Create BackupConfiguration:** + +Below is the YAML for `BackupConfiguration` CR to take application-level backup of the `sample-mysql` database that we have deployed earlier, + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: manifest-backup + - name: logical-backup +``` + +- `.spec.sessions[*].schedule` specifies that we want to backup at `5 minutes` interval. +- `.spec.target` refers to the targeted `sample-mysql` MySQL database that we created earlier. +- `.spec.sessions[*].addon.tasks[*].name[*]` specifies that both the `manifest-backup` and `logical-backup` tasks will be executed. + +Let's create the `BackupConfiguration` CR that we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/backupconfiguration.yaml +backupconfiguration.core.kubestash.com/sample-mysql-backup created +``` + +**Verify Backup Setup Successful** + +If everything goes well, the phase of the `BackupConfiguration` should be `Ready`. The `Ready` phase indicates that the backup setup is successful. Let's verify the `Phase` of the BackupConfiguration, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +sample-mysql-backup Ready 2m50s +``` + +Additionally, we can verify that the `Repository` specified in the `BackupConfiguration` has been created using the following command, + +```bash +$ kubectl get repo -n demo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-mysql-repo 0 0 B Ready 3m +``` + +KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/mysql` directory. + +**Verify CronJob:** + +It will also create a `CronJob` with the schedule specified in `spec.sessions[*].scheduler.schedule` field of `BackupConfiguration` CR. + +Verify that the `CronJob` has been created using the following command, + +```bash +$ kubectl get cronjob -n demo +NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE +trigger-sample-mysql-backup-frequent-backup */5 * * * * 0 2m45s 3m25s +``` + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +Run the following command to watch `BackupSession` CR, + +```bash +$ kubectl get backupsession -n demo -w + +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +sample-mysql-backup-frequent-backup-1724065200 BackupConfiguration sample-mysql-backup Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-mysql-backup` has been updated by the following command, + +```bash +$ kubectl get repository -n demo gcs-mysql-repo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-mysql-repo true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=gcs-demo-repo +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 sample-mysql-backup frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-09-03T10:25:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubestash.com/app-ref-kind: MySQL + kubestash.com/app-ref-name: sample-mysql + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: gcs-mysql-repo + annotations: + kubedb.com/db-version: 8.2.0 + name: gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: gcs-mysql-repo + uid: 1f5ba355-7f99-4b99-8bbf-9f9d4f31c52a + resourceVersion: "213010" + uid: 18cabb10-e594-4655-8763-3daa0872508e +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo + backupSession: sample-mysql-backup-frequent-backup-1725359100 + deletionPolicy: Delete + repository: gcs-mysql-repo + session: frequent-backup + snapshotID: 01J6VPN4TPHDFT1M9Q9YVGMTKF + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 7.393324414s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: f2ffd1bdb98563e15c46d8927d7239873ce7094132d959e12134688e06984736 + size: 3.657 MiB + uploaded: 706.081 KiB + size: 893.009 KiB + manifest: + driver: Restic + duration: 12.672292995s + integrity: true + path: repository/v1/frequent-backup/manifest + phase: Succeeded + resticStats: + - hostPath: /kubestash-tmp/manifest + id: ff99eb7ea769a365f7cdc83a252df610c262fc934ec0a3475499bbbb35ca6931 + size: 2.883 KiB + uploaded: 1.440 KiB + size: 3.788 KiB + conditions: + - lastTransitionTime: "2024-09-03T10:25:00Z" + message: Recent snapshot list updated successfully + reason: SuccessfullyUpdatedRecentSnapshotList + status: "True" + type: RecentSnapshotListUpdated + - lastTransitionTime: "2024-09-03T10:25:49Z" + message: Metadata uploaded to backend successfully + reason: SuccessfullyUploadedSnapshotMetadata + status: "True" + type: SnapshotMetadataUploaded + integrity: true + phase: Succeeded + size: 896.796 KiB + snapshotTime: "2024-09-03T10:25:00Z" + totalComponents: 2 +``` + +> KubeStash uses the `mysqldump` command to take backups of target MySQL databases. Therefore, the component name for logical backups is set as `dump`. +> KubeStash set component name as `manifest` for the `manifest backup` of MySQL databases. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/mysql/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/dep/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Restore + +In this section, we are going to restore the entire database from the backup that we have taken in the previous section. + +For this tutorial, we will restore the database in a separate namespace called `dev`. + +First, create the namespace by running the following command: + +```bash +$ kubectl create ns dev +namespace/dev created +``` + +#### Create RestoreSession: + +We need to create a RestoreSession CR. + +Below, is the contents of YAML file of the `RestoreSession` CR that we are going to create to restore the entire database. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + manifestOptions: + restoreNamespace: dev + mySQL: + db: true + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore + - name: manifest-restore +``` + +Here, + +- `.spec.manifestOptions.mySQL.db` specifies whether to restore the DB manifest or not. +- `.spec.dataSource.repository` specifies the Repository object that holds the backed up data. +- `.spec.dataSource.snapshot` specifies to restore from latest `Snapshot`. +- `.spec.addon.tasks[*]` specifies that both the `manifest-restore` and `logical-backup-restore` tasks. + +Let's create the RestoreSession CRD object we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/restoresession.yaml +restoresession.core.kubestash.com/sample-mysql-restore created +``` + +Once, you have created the `RestoreSession` object, KubeStash will create restore Job. Run the following command to watch the phase of the `RestoreSession` object, + +```bash +$ watch kubectl get restoresession -n demo +Every 2.0s: kubectl get restores... AppsCode-PC-03: Wed Aug 21 10:44:05 2024 + +NAME REPOSITORY FAILURE-POLICY PHASE DURATION AGE +sample-restore gcs-demo-repo Succeeded 3s 53s +``` +The `Succeeded` phase means that the restore process has been completed successfully. + +#### Verify Restored MySQL Manifest: + +In this section, we will verify whether the desired `MySQL` database manifest has been successfully applied to the cluster. + +```bash +$ kubectl get mysqls.kubedb.com -n dev +NAME VERSION STATUS AGE +sample-mysql 8.2.0 Ready 39m +``` + +The output confirms that the `MySQL` database has been successfully created with the same configuration as it had at the time of backup. + +#### Verify Restored Data: + +In this section, we are going to verify whether the desired data has been restored successfully. We are going to connect to the database server and check whether the database and the table we created earlier in the original database are restored. + +At first, check if the database has gone into `Ready` state by the following command, + +```bash +$ kubectl get my -n dev sample-mysql +NAME VERSION STATUS AGE +sample-mysql 8.2.0 Ready 4m +``` + +Now, find out the database `Pod` by the following command, + +```bash +$ kubectl get pods -n dev --selector="app.kubernetes.io/instance=sample-mysql" +NAME READY STATUS RESTARTS AGE +sample-mysql-0 2/2 Running 0 2m +sample-mysql-1 2/2 Running 0 2m +sample-mysql-2 2/2 Running 0 2m +``` + +And then copy the username and password of the `root` user to access into `mysql` shell. + +```bash +$ kubectl get secret -n dev sample-mysql-auth -o jsonpath='{.data.username}'| base64 -d +root + +$ kubectl get secret -n dev sample-mysql-auth -o jsonpath='{.data.password}'| base64 -d +QMm1hi0T*7QFz_yh +``` + +```bash +$ kubectl exec -it -n dev sample-mysql-0 -- mysql --user=root --password='QMm1hi0T*7QFz_yh' +Defaulted container "mysql" out of: mysql, mysql-coordinator, mysql-init (init) +mysql: [Warning] Using a password on the command line interface can be insecure. +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 243 +Server version: 8.2.0 MySQL Community Server - GPL + +Copyright (c) 2000, 2023, Oracle and/or its affiliates. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| information_schema | +| mysql | +| performance_schema | +| playground | +| sys | ++--------------------+ +5 rows in set (0.00 sec) + +mysql> SHOW TABLES IN playground; ++----------------------+ +| Tables_in_playground | ++----------------------+ +| equipment | ++----------------------+ +1 row in set (0.00 sec) + +mysql> SELECT * FROM playground.equipment; ++----+-------+-------+-------+ +| id | type | quant | color | ++----+-------+-------+-------+ +| 1 | slide | 2 | blue | ++----+-------+-------+-------+ +1 row in set (0.00 sec) + +mysql> exit +Bye +``` + +So, from the above output, we can see that the `playground` database and the `equipment` table we have created earlier in the original database and now, they are restored successfully. + +## Cleanup + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl delete backupconfigurations.core.kubestash.com -n demo sample-mysql-backup +kubectl delete backupstorage -n demo gcs-storage +kubectl delete secret -n demo gcs-secret +kubectl delete secret -n demo encrypt-secret +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention +kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-mysql +kubectl delete my -n demo sample-mysql +kubectl delete my -n dev sample-mysql +``` \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/auto-backup/examples/backupstorage.yaml b/docs/guides/mysql/backup/kubestash/auto-backup/examples/backupstorage.yaml new file mode 100644 index 0000000000..f5478706b6 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/auto-backup/examples/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: blueprint + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml b/docs/guides/mysql/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml new file mode 100644 index 0000000000..ba21426d0d --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml @@ -0,0 +1,41 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: mysql-customize-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + # ============== Blueprint for Backends of BackupConfiguration ================= + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + # ============== Blueprint for Sessions of BackupConfiguration ================= + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: ${schedule} + jobTemplate: + backoffLimit: 1 + repositories: + - name: ${repoName} + backend: gcs-backend + directory: ${namespace}/${targetName} + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup + params: + databases: ${targetedDatabases} \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml b/docs/guides/mysql/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml new file mode 100644 index 0000000000..9e9f1f76e5 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml @@ -0,0 +1,37 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: mysql-default-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: default-blueprint + backend: gcs-backend + directory: /default-blueprint + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/auto-backup/examples/retentionpolicy.yaml b/docs/guides/mysql/backup/kubestash/auto-backup/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/auto-backup/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql-2.yaml b/docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql-2.yaml new file mode 100644 index 0000000000..3d59b9a1a3 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql-2.yaml @@ -0,0 +1,25 @@ +apiVersion: kubedb.com/v1alpha2 +kind: MySQL +metadata: + name: sample-mysql-2 + namespace: demo + annotations: + blueprint.kubestash.com/name: mysql-customize-backup-blueprint + blueprint.kubestash.com/namespace: demo + variables.kubestash.com/schedule: "*/10 * * * *" + variables.kubestash.com/repoName: customize-blueprint + variables.kubestash.com/namespace: demo + variables.kubestash.com/targetName: sample-mysql-2 + variables.kubestash.com/targetedDatabases: mysql +spec: + version: "8.2.0" + replicas: 1 + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + terminationPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql.yaml b/docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql.yaml new file mode 100644 index 0000000000..8ca2d92ea6 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql.yaml @@ -0,0 +1,20 @@ +apiVersion: kubedb.com/v1alpha2 +kind: MySQL +metadata: + name: sample-mysql + namespace: demo + annotations: + blueprint.kubestash.com/name: mysql-default-backup-blueprint + blueprint.kubestash.com/namespace: demo +spec: + version: "8.2.0" + replicas: 1 + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + terminationPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/auto-backup/index.md b/docs/guides/mysql/backup/kubestash/auto-backup/index.md new file mode 100644 index 0000000000..3d1aae36b7 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/auto-backup/index.md @@ -0,0 +1,738 @@ +--- +title: MySQL Auto-Backup | KubeStash +description: Backup MySQL using KubeStash Auto-Backup +menu: + docs_{{ .version }}: + identifier: guides-mysql-backup-auto-backup-stashv2 + name: Auto-Backup + parent: guides-mysql-backup-stashv2 + weight: 30 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Backup MySQL using KubeStash Auto-Backup + +KubeStash can automatically be configured to backup any `MySQL` databases in your cluster. KubeStash enables cluster administrators to deploy backup `blueprints` ahead of time so database owners can easily backup any `MySQL` database with a few annotations. + +In this tutorial, we are going to show how you can configure a backup blueprint for `MySQL` databases in your cluster and backup them with a few annotations. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. +- Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). +- Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). +- Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- If you are not familiar with how KubeStash backup and restore MySQL databases, please check the following guide [here](/docs/guides/mysql/backup/kubestash/overview/index.md). + +You should be familiar with the following `KubeStash` concepts: + +- [BackupStorage](https://kubestash.com/docs/latest/concepts/crds/backupstorage/) +- [BackupConfiguration](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/) +- [BackupSession](https://kubestash.com/docs/latest/concepts/crds/backupsession/) +- [RestoreSession](https://kubestash.com/docs/latest/concepts/crds/restoresession/) +- [Addon](https://kubestash.com/docs/latest/concepts/crds/addon/) +- [Function](https://kubestash.com/docs/latest/concepts/crds/function/) +- [Task](https://kubestash.com/docs/latest/concepts/crds/addon/#task-specification) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +### Prepare Backend + +We are going to store our backed up data into a GCS bucket. We have to create a Secret with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). + +**Create Secret:** + +Let's create a secret called `gcs-secret` with access credentials to our desired GCS bucket, + +```bash +$ echo -n '' > GOOGLE_PROJECT_ID +$ cat /path/to/downloaded-sa-key.json > GOOGLE_SERVICE_ACCOUNT_JSON_KEY +$ kubectl create secret generic -n demo gcs-secret \ + --from-file=./GOOGLE_PROJECT_ID \ + --from-file=./GOOGLE_SERVICE_ACCOUNT_JSON_KEY +secret/gcs-secret created +``` + +**Create BackupStorage:** + +Now, create a `BackupStorage` using this secret. Below is the YAML of `BackupStorage` CR we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: blueprint + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete +``` + +Let's create the BackupStorage we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/auto-backup/examples/backupstorage.yaml +backupstorage.storage.kubestash.com/gcs-storage created +``` + +**Create RetentionPolicy:** + +Now, let's create a `RetentionPolicy` to specify how the old Snapshots should be cleaned up. + +Below is the YAML of the `RetentionPolicy` object that we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Let’s create the above `RetentionPolicy`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/auto-backup/examples/retentionpolicy.yaml +retentionpolicy.storage.kubestash.com/demo-retention created +``` + +**Create Secret:** + +We also need to create a secret with a `Restic` password for backup data encryption. + +Let's create a secret called `encrypt-secret` with the Restic password, + +```bash +$ echo -n 'changeit' > RESTIC_PASSWORD +$ kubectl create secret generic -n demo encrypt-secret \ + --from-file=./RESTIC_PASSWORD \ +secret "encrypt-secret" created +``` + +## Auto-backup with default configurations + +In this section, we are going to backup a `MySQL` database of `demo` namespace. We are going to use the default configurations which will be specified in the `Backup Blueprint` CR. + +**Prepare Backup Blueprint** + +A `BackupBlueprint` allows you to specify a template for the `Repository`,`Session` or `Variables` of `BackupConfiguration` in a Kubernetes native way. + +Now, we have to create a `BackupBlueprint` CR with a blueprint for `BackupConfiguration` object. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: mysql-default-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: default-blueprint + backend: gcs-backend + directory: /default-blueprint + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup +``` + +Here, + +- `.spec.backupConfigurationTemplate.backends[*].storageRef` refers our earlier created `gcs-storage` backupStorage. +- `.spec.backupConfigurationTemplate.sessions[*].schedule` specifies that we want to backup the database at `5 minutes` interval. + +Let's create the `BackupBlueprint` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml +backupblueprint.core.kubestash.com/mysql-default-backup-blueprint created +``` + +Now, we are ready to backup our `MySQL` databases using few annotations. + +**Create Database** + +Now, we are going to create an `MySQL` CR in demo namespace. Below is the YAML of the MySQL object that we are going to create, + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: MySQL +metadata: + name: sample-mysql + namespace: demo + annotations: + blueprint.kubestash.com/name: mysql-default-backup-blueprint + blueprint.kubestash.com/namespace: demo +spec: + version: "8.2.0" + replicas: 1 + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + terminationPolicy: WipeOut +``` + +Here, + +- `.spec.annotations.blueprint.kubestash.com/name: mysql-default-backup-blueprint` specifies the name of the `BackupBlueprint` that will use in backup. +- `.spec.annotations.blueprint.kubestash.com/namespace: demo` specifies the name of the `namespace` where the `BackupBlueprint` resides. + +Let's create the `MySQL` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql.yaml +mysql.kubedb.com/sample-mysql created +``` + +**Verify BackupConfiguration** + +If everything goes well, KubeStash should create a `BackupConfiguration` for our MySQL in demo namespace and the phase of that `BackupConfiguration` should be `Ready`. Verify the `BackupConfiguration` object by the following command, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +appbinding-sample-mysql Ready 2m50m +``` + +Now, let’s check the YAML of the `BackupConfiguration`. + +```bash +$ kubectl get backupconfiguration -n demo appbinding-sample-mysql -o yaml +``` + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + creationTimestamp: "2024-08-21T09:47:34Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + app.kubernetes.io/managed-by: kubestash.com + kubestash.com/invoker-name: mysql-default-backup-blueprint + kubestash.com/invoker-namespace: demo + name: appbinding-sample-mysql + namespace: demo + resourceVersion: "113911" + uid: eef4c853-4df6-4b5e-b462-977c9b2188c0 +spec: + backends: + - name: gcs-backend + retentionPolicy: + name: demo-retention + namespace: demo + storageRef: + name: gcs-storage + namespace: demo + sessions: + - addon: + name: mysql-addon + tasks: + - name: logical-backup + name: frequent-backup + repositories: + - backend: gcs-backend + directory: /default-blueprint + encryptionSecret: + name: encrypt-secret + namespace: demo + name: default-blueprint + scheduler: + jobTemplate: + backoffLimit: 1 + template: + controller: {} + metadata: {} + spec: + resources: {} + schedule: '*/5 * * * *' + sessionHistoryLimit: 3 + target: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo +``` + +Notice the `spec.backends`, `spec.sessions` and `spec.target` sections, KubeStash automatically resolved those info from the `BackupBluePrint` and created above `BackupConfiguration`. + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w + +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +appbinding-sample-mysql-frequent-backup-1724236500 BackupConfiguration appbinding-sample-mysql Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `default-blueprint` has been updated by the following command, + +```bash +$ kubectl get repository -n demo default-blueprint +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +default-blueprint true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=default-blueprint +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +default-blueprint-appbinding-sampleysql-frequent-backup-1724236500 default-blueprint frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubedb.com/db-version: ` +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo default-blueprint-appbinding-sampleysql-frequent-backup-1724236500 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-08-21T10:35:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubedb.com/db-version: 8.2.0 + kubestash.com/app-ref-kind: MySQL + kubestash.com/app-ref-name: sample-mysql + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: default-blueprint + name: default-blueprint-appbinding-sampleysql-frequent-backup-1724236500 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: default-blueprint + uid: 61e771fc-8262-480c-a9e7-3c5c11c8fd77 + resourceVersion: "118423" + uid: 27e2235a-22c1-449a-be92-c53506fe1fe4 +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo + backupSession: appbinding-sample-mysql-frequent-backup-1724236500 + deletionPolicy: Delete + repository: default-blueprint + session: frequent-backup + snapshotID: 01J6V48XS6QM489WPKX1MDD4W9 + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 6.692337543s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: b83d7a5577940d1c8f5bcda0630592c7d5a04168c272c0e7560bf7dacfe35ea8 + size: 3.657 MiB + uploaded: 121.343 KiB + size: 772.958 KiB + integrity: true + phase: Succeeded + size: 772.957 KiB + snapshotTime: "2024-08-21T10:35:00Z" + totalComponents: 1 +``` + +> KubeStash uses the `mysqldump` command to take backups of target MySQL databases. Therefore, the component name for `logical backups` is set as `dump`. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `/blueprint/default-blueprint/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/default-blueprintrepository/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Auto-backup with custom configurations + +In this section, we are going to backup a `MySQL` database of `demo` namespace. We are going to use the custom configurations which will be specified in the `BackupBlueprint` CR. + +**Prepare Backup Blueprint** + +A `BackupBlueprint` allows you to specify a template for the `Repository`,`Session` or `Variables` of `BackupConfiguration` in a Kubernetes native way. + +Now, we have to create a `BackupBlueprint` CR with a blueprint for `BackupConfiguration` object. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: mysql-customize-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + # ============== Blueprint for Backends of BackupConfiguration ================= + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + # ============== Blueprint for Sessions of BackupConfiguration ================= + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: ${schedule} + jobTemplate: + backoffLimit: 1 + repositories: + - name: ${repoName} + backend: gcs-backend + directory: ${namespace}/${targetName} + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup + params: + databases: ${targetedDatabases} +``` + +Note that we have used some variables (format: `${}`) in different fields. KubeStash will substitute these variables with values from the respective target’s annotations. You’re free to use any variables you like. + +Here, + +- `.spec.backupConfigurationTemplate.backends[*].storageRef` refers our earlier created `gcs-storage` backupStorage. +- `.spec.backupConfigurationTemplate.sessions[*]`: + - `.schedule` defines `${schedule}` variable, which determines the time interval for the backup. + - `.repositories[*].name` defines the `${repoName}` variable, which specifies the name of the backup `Repository`. + - `.repositories[*].directory` defines two variables, `${namespace}` and `${targetName}`, which are used to determine the path where the backup will be stored. + - `.addon.tasks[*]databases` defines `${targetedDatabases}` variable, which identifies list of databases to backup. + +Let's create the `BackupBlueprint` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml +backupblueprint.core.kubestash.com/mysql-customize-backup-blueprint created +``` + +Now, we are ready to backup our `MySQL` databases using few annotations. You can check available auto-backup annotations for a databases from [here](https://kubestash.com/docs/latest/concepts/crds/backupblueprint/). + +**Create Database** + +Now, we are going to create an `MySQL` CR in demo namespace. Below is the YAML of the MySQL object that we are going to create, + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: MySQL +metadata: + name: sample-mysql-2 + namespace: demo + annotations: + blueprint.kubestash.com/name: mysql-customize-backup-blueprint + blueprint.kubestash.com/namespace: demo + variables.kubestash.com/schedule: "*/10 * * * *" + variables.kubestash.com/repoName: customize-blueprint + variables.kubestash.com/namespace: demo + variables.kubestash.com/targetName: sample-mysql-2 + variables.kubestash.com/targetedDatabases: mysql +spec: + version: "8.2.0" + replicas: 1 + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + terminationPolicy: WipeOut +``` + +Notice the `metadata.annotations` field, where we have defined the annotations related to the automatic backup configuration. Specifically, we've set the `BackupBlueprint` name as `mysql-customize-backup-blueprint` and the namespace as `demo`. We have also provided values for the blueprint template variables, such as the backup `schedule`, `repositoryName`, `namespace`, `targetName`, and `targetedDatabases`. These annotations will be used to create a `BackupConfiguration` for this `MySQL` database. + +Let's create the `MySQL` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/auto-backup/examples/sample-mysql-2.yaml +mysql.kubedb.com/sample-mysql-2 created +``` + +**Verify BackupConfiguration** + +If everything goes well, KubeStash should create a `BackupConfiguration` for our MySQL in demo namespace and the phase of that `BackupConfiguration` should be `Ready`. Verify the `BackupConfiguration` object by the following command, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +appbinding-sample-mysql-2 Ready 2m50m +``` + +Now, let’s check the YAML of the `BackupConfiguration`. + +```bash +$ kubectl get backupconfiguration -n demo appbinding-sample-mysql-2 -o yaml +``` + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + creationTimestamp: "2024-08-21T12:55:38Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + app.kubernetes.io/managed-by: kubestash.com + kubestash.com/invoker-name: mysql-customize-backup-blueprint + kubestash.com/invoker-namespace: demo + name: appbinding-sample-mysql-2 + namespace: demo + resourceVersion: "129124" + uid: eb42b736-c9c9-4280-8379-bbb581790185 +spec: + backends: + - name: gcs-backend + retentionPolicy: + name: demo-retention + namespace: demo + storageRef: + name: gcs-storage + namespace: demo + sessions: + - addon: + name: mysql-addon + tasks: + - name: logical-backup + params: + databases: mysql + name: frequent-backup + repositories: + - backend: gcs-backend + directory: demo/sample-mysql-2 + encryptionSecret: + name: encrypt-secret + namespace: demo + name: customize-blueprint + scheduler: + jobTemplate: + backoffLimit: 1 + template: + controller: {} + metadata: {} + spec: + resources: {} + schedule: '*/10 * * * *' + sessionHistoryLimit: 3 + target: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql-2 + namespace: demo +``` + +Notice the `spec.backends`, `spec.sessions` and `spec.target` sections, KubeStash automatically resolved those info from the `BackupBluePrint` and created above `BackupConfiguration`. + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w + +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +appbinding-sample-mysql-2-frequent-backup-1725007200 BackupConfiguration appbinding-sample-mysql-2 Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `customize-blueprint` has been updated by the following command, + +```bash +$ kubectl get repository -n demo customize-blueprint +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +customize-blueprint true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=customize-blueprint +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +customize-blueprint-appbinding-sql-2-frequent-backup-1725007200 customize-blueprint frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubedb.com/db-version: ` +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo customize-blueprint-appbinding-sql-2-frequent-backup-1725007200 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-08-21T10:35:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubedb.com/db-version: 8.2.0 + kubestash.com/app-ref-kind: MySQL + kubestash.com/app-ref-name: sample-mysql + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: customize-blueprint + name: customize-blueprint-appbinding-sql-2-frequent-backup-1725007200 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: customize-blueprint + uid: 61e771fc-8262-480c-a9e7-3c5c11c8fd77 + resourceVersion: "118423" + uid: 27e2235a-22c1-449a-be92-c53506fe1fe4 +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql-2 + namespace: demo + backupSession: appbinding-sample-mysql-2-frequent-backup-1725007200 + deletionPolicy: Delete + repository: customize-blueprint + session: frequent-backup + snapshotID: 01J6V48XZW2BJ02GSS4YBW3TWX + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 6.692337543s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: b83d7a5577940d1c8f5bcda0630592c7d5a04168c272c0e7560bf7dacfe35ea8 + size: 3.657 MiB + uploaded: 121.343 KiB + size: 772.958 KiB + integrity: true + phase: Succeeded + size: 772.957 KiB + snapshotTime: "2024-08-21T10:35:00Z" + totalComponents: 1 +``` + +> KubeStash uses the `mysqldump` command to take backups of target MySQL databases. Therefore, the component name for `logical backups` is set as `dump`. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `/blueprint/custom-blueprint/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/custom-blueprint/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Cleanup + +To cleanup the resources crated by this tutorial, run the following commands, + +```bash +kubectl delete backupblueprints.core.kubestash.com -n demo mysql-default-backup-blueprint +kubectl delete backupblueprints.core.kubestash.com -n demo mysql-customize-backup-blueprint +kubectl delete backupstorage -n demo gcs-storage +kubectl delete secret -n demo gcs-secret +kubectl delete secret -n demo encrypt-secret +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention +kubectl delete my -n demo sample-mysql +kubectl delete my -n demo sample-mysql-2 +``` \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/backup/multiple-backends.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/backup/multiple-backends.yaml new file mode 100644 index 0000000000..af81b8cf1b --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/backup/multiple-backends.yaml @@ -0,0 +1,49 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + - name: s3-mysql-repo + backend: s3-backend + directory: /mysql-copy + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/backup/passing-args.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/backup/passing-args.yaml new file mode 100644 index 0000000000..5a537a5228 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/backup/passing-args.yaml @@ -0,0 +1,38 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup + params: + args: --set-gtid-purged=OFF \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/backup/passing-databases.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/backup/passing-databases.yaml new file mode 100644 index 0000000000..4483696ee8 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/backup/passing-databases.yaml @@ -0,0 +1,38 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup + params: + databases: db1,db2,db3 \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/backup/resources-limit.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/backup/resources-limit.yaml new file mode 100644 index 0000000000..5b247de5d2 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/backup/resources-limit.yaml @@ -0,0 +1,46 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup + diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/backup/specific-user.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/backup/specific-user.yaml new file mode 100644 index 0000000000..07cf18c8e4 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/backup/specific-user.yaml @@ -0,0 +1,41 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/common/gcs-backupstorage.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/common/gcs-backupstorage.yaml new file mode 100644 index 0000000000..6ab3df02ac --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/common/gcs-backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/common/retentionpolicy.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/common/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/common/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/common/s3-backupstorage.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/common/s3-backupstorage.yaml new file mode 100644 index 0000000000..a0f1c3ade3 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/common/s3-backupstorage.yaml @@ -0,0 +1,19 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: s3-storage + namespace: demo +spec: + storage: + provider: s3 + s3: + bucket: kubestash + region: us-east-1 + endpoint: us-east-1.linodeobjects.com + secretName: s3-secret + prefix: demo + usagePolicy: + allowedNamespaces: + from: All + default: false + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/common/sample-mysql.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/common/sample-mysql.yaml new file mode 100644 index 0000000000..89aa071feb --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/common/sample-mysql.yaml @@ -0,0 +1,18 @@ +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: sample-mysql + namespace: demo +spec: + version: "8.2.0" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/restore/passing-args.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/restore/passing-args.yaml new file mode 100644 index 0000000000..044a22fde9 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/restore/passing-args.yaml @@ -0,0 +1,23 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore + params: + args: --one-database=testdb \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/restore/resources-limit.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/restore/resources-limit.yaml new file mode 100644 index 0000000000..d4d2924325 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/restore/resources-limit.yaml @@ -0,0 +1,30 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup-restore \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/restore/specific-snapshot.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/restore/specific-snapshot.yaml new file mode 100644 index 0000000000..861911e5e7 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/restore/specific-snapshot.yaml @@ -0,0 +1,21 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725257849 + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore diff --git a/docs/guides/mysql/backup/kubestash/customization/examples/restore/specific-user.yaml b/docs/guides/mysql/backup/kubestash/customization/examples/restore/specific-user.yaml new file mode 100644 index 0000000000..1c3944573f --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/examples/restore/specific-user.yaml @@ -0,0 +1,26 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup-restore diff --git a/docs/guides/mysql/backup/kubestash/customization/index.md b/docs/guides/mysql/backup/kubestash/customization/index.md new file mode 100644 index 0000000000..2b8097d149 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/customization/index.md @@ -0,0 +1,444 @@ +--- +title: MySQL Backup Customization | KubeStash +description: Customizing MySQL Backup and Restore process with KubeStash +menu: + docs_{{ .version }}: + identifier: guides-mysql-backup-customization-stashv2 + name: Customizing Backup & Restore Process + parent: guides-mysql-backup-stashv2 + weight: 50 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Customizing Backup and Restore Process + +KubeStash provides rich customization supports for the backup and restore process to meet the requirements of various cluster configurations. This guide will show you some examples of these customizations. + +## Customizing Backup Process + +In this section, we are going to show you how to customize the backup process. Here, we are going to show some examples of providing arguments to the backup process, running the backup process as a specific user, etc. + +### Passing targeted databases to the backup process + +KubeStash MySQL addon uses the [mysqldump](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html) for backup. Addon has implemented a `databases` params which indicates your targeted backup databases. + +The below example shows how you can pass the `--databases` option during backup. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup + params: + databases: db1,db2,db3 +``` + +> **WARNING**: Make sure that your provides databases has been created before taking backup. + +Here, +- `addon.tasks[*].databases` options indicates targeted databases. By default `myaql-addon` add `--alll-databases` options during backup. If you want to backup all databases keep the `databases` params empty. + +### Passing arguments to the backup process + +KubeStash MySQL addon uses [mysqldump](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html) for backup. You can pass arguments to the `mysqldump` supported options through `args` param under `addon.tasks[*].params` section. + +The below example shows how you can pass the `--set-gtid-purged=OFF` options during backup. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup + params: + args: --set-gtid-purged=OFF +``` + + +### Using multiple backends + +You can configure multiple backends within a single `backupConfiguration`. To back up the same data to different backends, such as S3 and GCS, declare each backend in the `.spe.backends` section. Then, reference these backends in the `.spec.sessions[*].repositories` section. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + - name: s3-mysql-repo + backend: s3-backend + directory: /mysql-copy + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup +``` + +### Running backup job as a specific user + +If your cluster requires running the backup job as a specific user, you can provide `securityContext` under `addon.jobTemplate.spec.securityContext` section. The below example shows how you can run the backup job as the `root` user. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup +``` + +### Specifying Memory/CPU limit/request for the backup job + +If you want to specify the Memory/CPU limit/request for your backup job, you can specify `resources` field under `addon.jobTemplate.spec` section. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup +``` + +> You can configure additional runtime settings for backup jobs within the `addon.jobTemplate.spec` sections. For further details, please refer to the [reference](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/#podtemplate-spec). + +## Customizing Restore Process + +KubeStash also uses `mysql` during the restore process. In this section, we are going to show how you can pass arguments to the restore process, restore a specific snapshot, run restore job as a specific user, etc. + + +### Passing arguments to the restore process + +Similar to the backup process, you can pass arguments to the restore process through the `args` params under `addon.tasks[*].params` section. + +This example will restore data from database `testdb` only. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup + params: + args: --one-database=testdb +``` + +### Restore specific snapshot + +You can also restore a specific snapshot. At first, list the available snapshot as bellow, + +```bash +➤ kubectl get snapshots.storage.kubestash.com -n demo -l=kubestash.com/repo-name=gcs-mysql-repo +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725257849 gcs-mysql-repo frequent-backup 2024-09-02T06:18:01Z Delete Succeeded 15m +gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725258000 gcs-mysql-repo frequent-backup 2024-09-02T06:20:00Z Delete Succeeded 13m +gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725258300 gcs-mysql-repo frequent-backup 2024-09-02T06:25:00Z Delete Succeeded 8m34s +gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725258600 gcs-mysql-repo frequent-backup 2024-09-02T06:30:00Z Delete Succeeded 3m34s +``` + +The below example shows how you can pass a specific snapshot name in `.dataSource` section. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore + params: + args: --one-database=testdb +``` + + +### Running restore job as a specific user + +Similar to the backup process under the `addon.jobTemplate.spec.` you can provide `securityContext` to run the restore job as a specific user. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup-restore +``` + +### Specifying Memory/CPU limit/request for the restore job + +Similar to the backup process, you can also provide `resources` field under the `addon.jobTemplate.spec.resources` section to limit the Memory/CPU for your restore job. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup-restore +``` + +> You can configure additional runtime settings for restore jobs within the `addon.jobTemplate.spec` sections. For further details, please refer to the [reference](https://kubestash.com/docs/latest/concepts/crds/restoresession/#podtemplate-spec). \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/logical/examples/backupconfiguration.yaml b/docs/guides/mysql/backup/kubestash/logical/examples/backupconfiguration.yaml new file mode 100644 index 0000000000..0ca97ba748 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/logical/examples/backupconfiguration.yaml @@ -0,0 +1,36 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/logical/examples/backupstorage.yaml b/docs/guides/mysql/backup/kubestash/logical/examples/backupstorage.yaml new file mode 100644 index 0000000000..6ab3df02ac --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/logical/examples/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/logical/examples/restored-mysql.yaml b/docs/guides/mysql/backup/kubestash/logical/examples/restored-mysql.yaml new file mode 100644 index 0000000000..e1131b2f2e --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/logical/examples/restored-mysql.yaml @@ -0,0 +1,20 @@ +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: restored-mysql + namespace: demo +spec: + init: + waitForInitialRestore: true + version: "8.2.0" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/logical/examples/restoresession.yaml b/docs/guides/mysql/backup/kubestash/logical/examples/restoresession.yaml new file mode 100644 index 0000000000..2438d4533a --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/logical/examples/restoresession.yaml @@ -0,0 +1,21 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore diff --git a/docs/guides/mysql/backup/kubestash/logical/examples/retentionpolicy.yaml b/docs/guides/mysql/backup/kubestash/logical/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/logical/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/logical/examples/sample-mysql.yaml b/docs/guides/mysql/backup/kubestash/logical/examples/sample-mysql.yaml new file mode 100644 index 0000000000..89aa071feb --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/logical/examples/sample-mysql.yaml @@ -0,0 +1,18 @@ +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: sample-mysql + namespace: demo +spec: + version: "8.2.0" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/logical/index.md b/docs/guides/mysql/backup/kubestash/logical/index.md new file mode 100644 index 0000000000..0419049bb8 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/logical/index.md @@ -0,0 +1,756 @@ +--- +title: Backup & Restore MySQL | KubeStash +description: Backup MySQL database using KubeStash +menu: + docs_{{ .version }}: + identifier: guides-mysql-logical-backup-stashv2 + name: Logical Backup + parent: guides-mysql-backup-stashv2 + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Backup and Restore MySQL database using KubeStash + +KubeStash allows you to backup and restore `MySQL` databases. It supports backups for `MySQL` instances running in Standalone, Group Replication, and InnoDB cluster configurations. KubeStash makes managing your `MySQL` backups and restorations more straightforward and efficient. + +This guide will give you how you can take backup and restore your `MySQL` databases using `Kubestash`. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. +- Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). +- Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). +- Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- If you are not familiar with how KubeStash backup and restore MySQL databases, please check the following guide [here](/docs/guides/mysql/backup/kubestash/overview/index.md). + +You should be familiar with the following `KubeStash` concepts: + +- [BackupStorage](https://kubestash.com/docs/latest/concepts/crds/backupstorage/) +- [BackupConfiguration](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/) +- [BackupSession](https://kubestash.com/docs/latest/concepts/crds/backupsession/) +- [RestoreSession](https://kubestash.com/docs/latest/concepts/crds/restoresession/) +- [Addon](https://kubestash.com/docs/latest/concepts/crds/addon/) +- [Function](https://kubestash.com/docs/latest/concepts/crds/function/) +- [Task](https://kubestash.com/docs/latest/concepts/crds/addon/#task-specification) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/guides/mysql/backup/kubestash/logical/examples](docs/guides/mysql/backup/kubestash/logical/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + +## Backup MySQL + +KubeStash supports backups for `MySQL` instances across different configurations, including Standalone, Group Replication, and InnoDB Cluster setups. In this demonstration, we'll focus on a `MySQL` database using Group Replication. The backup and restore process is similar for Standalone and InnoDB Cluster configurations as well. + +This section will demonstrate how to backup a `MySQL` database. Here, we are going to deploy a `MySQL` database using KubeDB. Then, we are going to backup this database into a `GCS` bucket. Finally, we are going to restore the backup up data into another `MySQL` database. + +### Deploy Sample MySQL Database + +Let's deploy a sample `MySQL` database and insert some data into it. + +**Create MySQL CR:** + +Below is the YAML of a sample `MySQL` CRD that we are going to create for this tutorial: + +```yaml +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: sample-mysql + namespace: demo +spec: + version: "8.2.0" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + deletionPolicy: WipeOut +``` + +Here, +- `.spec.topology` specifies about the clustering configuration of MySQL. +- `.Spec.topology.mode` specifies the mode of MySQL Cluster. During the demonstration we consider to use `GroupReplication`. + +Create the above `MySQL` CR, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/sample-mysql.yaml +mysql.kubedb.com/sample-mysql created +``` + +KubeDB will deploy a MySQL database according to the above specification. It will also create the necessary `Secrets` and `Services` to access the database. + +Let's check if the database is ready to use, + +```bash +$ kubectl get mysqls.kubedb.com -n demo +NAME VERSION STATUS AGE +sample-mysql 8.2.0 Ready 4m22s +``` + +The database is `Ready`. Verify that KubeDB has created a `Secret` and a `Service` for this database using the following commands, + +```bash +$ kubectl get secret -n demo -l=app.kubernetes.io/instance=sample-mysql +NAME TYPE DATA AGE +sample-mysql-auth Opaque 2 4m58s + +$ kubectl get service -n demo -l=app.kubernetes.io/instance=sample-mysql +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +sample-mysql ClusterIP 10.96.55.61 3306/TCP 97s +sample-mysql-pods ClusterIP None 3306/TCP 97s +sample-mysql-standby ClusterIP 10.96.211.186 3306/TCP 97 +``` + +Here, we have to use service `sample-mysql` and secret `sample-mysql-auth` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/mysql/concepts/appbinding/index.md) CR that holds the necessary information to connect with the database. + +**Verify AppBinding:** + +Verify that the `AppBinding` has been created successfully using the following command, + +```bash +$ kubectl get appbindings -n demo +NAME AGE +sample-mysql 9m24s +``` + +Let's check the YAML of the above `AppBinding`, + +```bash +$ kubectl get appbindings -n demo sample-mysql -o yaml +``` + +```yaml +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: sample-mysql + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: mysqls.kubedb.com + name: sample-mysql + namespace: demo + ownerReferences: + - apiVersion: kubedb.com/v1 + blockOwnerDeletion: true + controller: true + kind: MySQL + name: sample-mysql + uid: edde3e8b-7775-4f91-85a9-4ba4b96315f7 + resourceVersion: "5126" + uid: 86c9a149-f8ab-44c4-947f-5f9b402aad6c +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo + clientConfig: + service: + name: sample-mysql + path: / + port: 3306 + scheme: tcp + url: tcp(sample-mysql.demo.svc:3306)/ + ... + ... + secret: + name: sample-mysql-auth + type: kubedb.com/mysql + version: 8.2.0 +``` + +KubeStash uses the `AppBinding` CR to connect with the target database. It requires the following two fields to set in AppBinding's `.spec` section. + +- `.spec.clientConfig.service.name` specifies the name of the Service that connects to the database. +- `.spec.secret` specifies the name of the Secret that holds necessary credentials to access the database. +- `spec.type` specifies the types of the app that this AppBinding is pointing to. KubeDB generated AppBinding follows the following format: `/`. + +**Insert Sample Data:** + +Now, we are going to exec into the database pod and create some sample data. At first, find out the database `Pod` using the following command, + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-mysql" +NAME READY STATUS RESTARTS AGE +sample-mysql-0 2/2 Running 0 2m41s +sample-mysql-1 2/2 Running 0 2m35s +sample-mysql-2 2/2 Running 0 2m29s +``` + +And copy the username and password of the `root` user to access into `mysql` shell. + +```bash +$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.username}'| base64 -d +root⏎ + +$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.password}'| base64 -d +DZfmUZd14fNEEOU4⏎ +``` + +Now, Lets exec into the `Pod` to enter into `mysql` shell and create a database and a table, + +```bash +$ kubectl exec -it -n demo sample-mysql-0 -- mysql --user=root --password=DZfmUZd14fNEEOU4 +Defaulted container "mysql" out of: mysql, mysql-init (init) +mysql: [Warning] Using a password on the command line interface can be insecure. +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 977 +Server version: 8.2.0 MySQL Community Server - GPL + +Copyright (c) 2000, 2023, Oracle and/or its affiliates. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> CREATE DATABASE playground; +Query OK, 1 row affected (0.01 sec) + +mysql> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| information_schema | +| mysql | +| performance_schema | +| playground | +| sys | ++--------------------+ +5 rows in set (0.00 sec) + +mysql> CREATE TABLE playground.equipment ( id INT NOT NULL AUTO_INCREMENT, type VARCHAR(50), quant INT, color VARCHAR(25), PRIMARY KEY(id)); +Query OK, 0 rows affected (0.01 sec) + +mysql> SHOW TABLES IN playground; ++----------------------+ +| Tables_in_playground | ++----------------------+ +| equipment | ++----------------------+ +1 row in set (0.01 sec) + +mysql> INSERT INTO playground.equipment (type, quant, color) VALUES ("slide", 2, "blue"); +Query OK, 1 row affected (0.01 sec) + +mysql> SELECT * FROM playground.equipment; ++----+-------+-------+-------+ +| id | type | quant | color | ++----+-------+-------+-------+ +| 1 | slide | 2 | blue | ++----+-------+-------+-------+ +1 row in set (0.00 sec) + +mysql> exit +Bye +``` + +Now, we are ready to backup the database. + +### Prepare Backend + +We are going to store our backed up data into a GCS bucket. We have to create a Secret with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). + +**Create Secret:** + +Let's create a secret called `gcs-secret` with access credentials to our desired GCS bucket, + +```bash +$ echo -n '' > GOOGLE_PROJECT_ID +$ cat /path/to/downloaded-sa-key.json > GOOGLE_SERVICE_ACCOUNT_JSON_KEY +$ kubectl create secret generic -n demo gcs-secret \ + --from-file=./GOOGLE_PROJECT_ID \ + --from-file=./GOOGLE_SERVICE_ACCOUNT_JSON_KEY +secret/gcs-secret created +``` + +**Create BackupStorage:** + +Now, create a `BackupStorage` using this secret. Below is the YAML of `BackupStorage` CR we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete +``` + +Let's create the BackupStorage we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/backupstorage.yaml +backupstorage.storage.kubestash.com/gcs-storage created +``` + +Now, we are ready to backup our database to our desired backend. + +**Create RetentionPolicy:** + +Now, let's create a `RetentionPolicy` to specify how the old Snapshots should be cleaned up. + +Below is the YAML of the `RetentionPolicy` object that we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Let’s create the above `RetentionPolicy`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/retentionpolicy.yaml +retentionpolicy.storage.kubestash.com/demo-retention created +``` + +### Backup + +We have to create a `BackupConfiguration` targeting respective `sample-mysql` MySQL database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. + +At first, we need to create a secret with a Restic password for backup data encryption. + +**Create Secret:** + +Let's create a secret called `encrypt-secret` with the Restic password, + +```bash +$ echo -n 'changeit' > RESTIC_PASSWORD +$ kubectl create secret generic -n demo encrypt-secret \ + --from-file=./RESTIC_PASSWORD \ +secret "encrypt-secret" created +``` + +**Create BackupConfiguration:** + +Below is the YAML for `BackupConfiguration` CR to backup the `sample-mysql` database that we have deployed earlier, + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup +``` + +- `.spec.sessions[*].schedule` specifies that we want to backup the database at `5 minutes` interval. +- `.spec.target` refers to the targeted `sample-mysql` MySQL database that we created earlier. + +Let's create the `BackupConfiguration` CR that we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/backupconfiguration.yaml +backupconfiguration.core.kubestash.com/sample-mysql-backup created +``` + +**Verify Backup Setup Successful** + +If everything goes well, the phase of the `BackupConfiguration` should be `Ready`. The `Ready` phase indicates that the backup setup is successful. Let's verify the `Phase` of the BackupConfiguration, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +sample-mysql-backup Ready 2m50s +``` + +Additionally, we can verify that the `Repository` specified in the `BackupConfiguration` has been created using the following command, + +```bash +$ kubectl get repo -n demo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-mysql-repo 0 0 B Ready 3m +``` + +KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/mysql` directory. + +**Verify CronJob:** + +It will also create a `CronJob` with the schedule specified in `spec.sessions[*].scheduler.schedule` field of `BackupConfiguration` CR. + +Verify that the `CronJob` has been created using the following command, + +```bash +$ kubectl get cronjob -n demo +NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE +trigger-sample-mysql-backup-frequent-backup */5 * * * * 0 2m45s 3m25s +``` + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w + +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +sample-mysql-backup-frequent-backup-1724065200 BackupConfiguration sample-mysql-backup Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-mysql-backup` has been updated by the following command, + +```bash +$ kubectl get repository -n demo gcs-mysql-repo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-mysql-repo true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=gcs-demo-repo +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 sample-mysql-backup frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-08-19T12:10:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubestash.com/app-ref-kind: MySQL + kubestash.com/app-ref-name: sample-mysql + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: gcs-mysql-repo + annotations: + kubedb.com/db-version: 8.2.0 + name: gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: gcs-mysql-repo + uid: 036a2605-0dcc-43bd-a921-363d5c3e8cf0 + resourceVersion: "33742" + uid: d7f14919-4c31-4b46-84e9-0a75f303ad92 +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo + backupSession: sample-mysql-backup-frequent-backup-1724065200 + deletionPolicy: Delete + repository: gcs-mysql-repo + session: frequent-backup + snapshotID: 01J6V4P4J0R33C8EG0JWK82118 + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 7.472499073s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: fbab3af5c38f51b7aa9096799d8ce4b71ea0092dd8297526fed0adbd9f62f3f1 + size: 3.657 MiB + uploaded: 1.036 MiB + size: 1.456 MiB + ... +``` + +> KubeStash uses the `mysqldump` command to take backups of target MySQL databases. Therefore, the component name for logical backups is set as `dump`. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/mysql/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/dep/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Restore + +In this section, we are going to restore the database from the backup we have taken in the previous section. We are going to deploy a new database and initialize it from the backup. + +#### Deploy Restored Database: + +Now, we have to deploy the restored database similarly as we have deployed the original `sample-mysql` database. However, this time there will be the following differences: + +- We are going to specify `.spec.init.waitForInitialRestore` field that tells KubeDB to wait for first restore to complete before marking this database is ready to use. + +Below is the YAML for `MySQL` CRD we are going deploy to initialize from backup, + +```yaml +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: restored-mysql + namespace: demo +spec: + init: + waitForInitialRestore: true + version: "8.2.0" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + deletionPolicy: WipeOut +``` + +Let's create the above database, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/restored-mysql.yaml +mysql.kubedb.com/restored-mysql created +``` + +If you check the database status, you will see it is stuck in `Provisioning` state. + +```bash +$ kubectl get my -n demo restored-mysql +NAME VERSION STATUS AGE +restored-mysql 8.2.0 Provisioning 61s +``` + +#### Create RestoreSession: + +Now, we need to create a RestoreSession CRD pointing to targeted `MySQL` database. + +Below, is the contents of YAML file of the `RestoreSession` object that we are going to create to restore backed up data into the newly created database provisioned by MySQL object named `restored-mysql`. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore +``` + +Here, + +- `.spec.target` refers to the newly created `restored-mysql` MySQL object to where we want to restore backup data. +- `.spec.dataSource.repository` specifies the Repository object that holds the backed up data. +- `.spec.dataSource.snapshot` specifies to restore from latest `Snapshot`. + +Let's create the RestoreSession CRD object we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/restoresession.yaml +restoresession.core.kubestash.com/sample-mysql-restore created +``` + +Once, you have created the `RestoreSession` object, KubeStash will create restore Job. Run the following command to watch the phase of the `RestoreSession` object, + +```bash +$ watch kubectl get restoresession -n demo +Every 2.0s: kubectl get restores... AppsCode-PC-03: Wed Aug 21 10:44:05 2024 + +NAME REPOSITORY FAILURE-POLICY PHASE DURATION AGE +sample-restore gcs-demo-repo Succeeded 3s 53s +``` + +The `Succeeded` phase means that the restore process has been completed successfully. + + +#### Verify Restored Data: + +In this section, we are going to verify whether the desired data has been restored successfully. We are going to connect to the database server and check whether the database and the table we created earlier in the original database are restored. + +At first, check if the database has gone into `Ready` state by the following command, + +```bash +$ kubectl get my -n demo restored-mysql +NAME VERSION STATUS AGE +restored-mysql 8.2.0 Ready 34m +``` + +Now, find out the database `Pod` by the following command, + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=restored-mysql" +NAME READY STATUS RESTARTS AGE +restored-mysql-0 1/1 Running 0 39m +``` + +And then copy the user name and password of the `root` user to access into `mysql` shell. + +```bash +$ kubectl get secret -n demo restored-mysql-auth -o jsonpath='{.data.username}'| base64 -d +root + +$ kubectl get secret -n demo restored-mysql-auth -o jsonpath='{.data.password}'| base64 -d +QMm1hi0T*7QFz_yh +``` + +Now, let's exec into the Pod to enter into `mysql` shell and verify restored data, + +```bash +$ kubectl exec -it -n demo restored-mysql-0 -- mysql --user=root --password='QMm1hi0T*7QFz_yh' +Defaulted container "mysql" out of: mysql, mysql-coordinator, mysql-init (init) +mysql: [Warning] Using a password on the command line interface can be insecure. +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 243 +Server version: 8.2.0 MySQL Community Server - GPL + +Copyright (c) 2000, 2023, Oracle and/or its affiliates. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| information_schema | +| mysql | +| performance_schema | +| playground | +| sys | ++--------------------+ +5 rows in set (0.00 sec) + +mysql> SHOW TABLES IN playground; ++----------------------+ +| Tables_in_playground | ++----------------------+ +| equipment | ++----------------------+ +1 row in set (0.00 sec) + +mysql> SELECT * FROM playground.equipment; ++----+-------+-------+-------+ +| id | type | quant | color | ++----+-------+-------+-------+ +| 1 | slide | 2 | blue | ++----+-------+-------+-------+ +1 row in set (0.00 sec) + +mysql> exit +Bye +``` + +So, from the above output, we can see that the `playground` database and the `equipment` table we have created earlier in the original database and now, they are restored successfully. + +## Cleanup + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl delete backupconfigurations.core.kubestash.com -n demo sample-mysql-backup +kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-mysql +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention +kubectl delete backupstorage -n demo gcs-storage +kubectl delete secret -n demo gcs-secret +kubectl delete secret -n demo encrypt-secret +kubectl delete my -n demo restored-mysql +kubectl delete my -n demo sample-mysql +``` \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/overview/images/backup_overview.svg b/docs/guides/mysql/backup/kubestash/overview/images/backup_overview.svg new file mode 100644 index 0000000000..628550b3b2 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/overview/images/backup_overview.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/overview/images/kubedb_plus_kubestash.svg b/docs/guides/mysql/backup/kubestash/overview/images/kubedb_plus_kubestash.svg new file mode 100644 index 0000000000..380d92d969 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/overview/images/kubedb_plus_kubestash.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/guides/mysql/backup/kubestash/overview/images/restore_overview.svg b/docs/guides/mysql/backup/kubestash/overview/images/restore_overview.svg new file mode 100644 index 0000000000..7f20b5ee92 --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/overview/images/restore_overview.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/guides/mysql/backup/kubestash/overview/index.md b/docs/guides/mysql/backup/kubestash/overview/index.md new file mode 100644 index 0000000000..9a724c314a --- /dev/null +++ b/docs/guides/mysql/backup/kubestash/overview/index.md @@ -0,0 +1,98 @@ +--- +title: Backup & Restore MySQL Overview +menu: + docs_{{ .version }}: + identifier: guides-mysql-backup-kubestash-overview-stashv2 + name: Overview + parent: guides-mysql-backup-stashv2 + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +{{< notice type="warning" message="Please install [KubeStash](https://kubestash.com/docs/latest/setup/install/kubestash/) to try this feature. Database backup with KubeStash is already included in the KubeDB license. So, you don't need a separate license for KubeStash." >}} + +# MySQL Backup & Restore Overview + +KubeDB also uses [KubeStash](https://kubestash.com) to backup and restore databases. KubeStash by AppsCode is a cloud native data backup and recovery solution for Kubernetes workloads and databases. KubeStash utilizes [restic](https://github.com/restic/restic) to securely backup stateful applications to any cloud or on-prem storage backends (for example, S3, GCS, Azure Blob storage, Minio, NetApp, Dell EMC etc.). + +
+  KubeDB + KubeStash +
Fig: Backup KubeDB Databases Using KubeStash
+
+ +## How Backup Works + +The following diagram shows how KubeStash takes backup of a `MySQL` database. Open the image in a new tab to see the enlarged version. + +
+  MySQL Backup Overview +
Fig: MySQL Backup Overview
+
+ +The backup process consists of the following steps: + +1. At first, a user creates a `Secret`. This secret holds the credentials to access the backend where the backed up data will be stored. + +2. Then, she creates a `BackupStorage` custom resource that specifies the backend information, along with the `Secret` containing the credentials needed to access the backend. + +3. KubeStash operator watches for `BackupStorage` custom resources. When it finds a `BackupStorage` object, it initializes the `BackupStorage` by uploading the `metadata.yaml` file to the specified backend. + +4. Next, she creates a `BackupConfiguration` custom resource that specifies the target database, addon information (including backup tasks), backup schedules, storage backends for storing the backup data, and other additional settings. + +5. KubeStash operator watches for `BackupConfiguration` objects. + +6. Once the KubeStash operator finds a `BackupConfiguration` object, it creates `Repository` with the information specified in the `BackupConfiguration`. + +7. KubeStash operator watches for `Repository` custom resources. When it finds the `Repository` object, it Initializes `Repository` by uploading `repository.yaml` file into the `spec.sessions[*].repositories[*].directory` path specified in `BackupConfiguration`. + +8. Then, it creates a `CronJob` for each session with the schedule specified in `BackupConfiguration` to trigger backup periodically. + +9. KubeStash operator triggers an instant backup as soon as the `BackupConfiguration` is ready. Backups are otherwise triggered by the `CronJob` based on the specified schedule. + +10. KubeStash operator watches for `BackupSession` custom resources. + +11. When it finds a `BackupSession` object, it creates a `Snapshot` custom resource for each `Repository` specified in the `BackupConfiguration`. + +12. Then it resolves the respective `Addon` and `Function` and prepares backup `Job` definition. + +13. Then, it creates the `Job` to backup the targeted `MySQL` database. + +14. The backup `Job` reads necessary information (e.g. auth secret, port) to connect with the database from the `AppBinding` CR. It also reads backend information and access credentials from `BackupStorage` CR, Storage `Secret` and `Repository` path respectively. + +15. Then, the `Job` dumps the targeted `MySQL` database and uploads the output to the backend. KubeStash pipes the output of dump command to uploading process. Hence, backup `Job` does not require a large volume to hold the entire dump output. + +16. After the backup process is completed, the backup `Job` updates the `status.components[dump]` field of the `Snapshot` resources with backup information of the target `MySQL` database. + +## How Restore Process Works + +The following diagram shows how KubeStash restores backed up data into a `MySQL` database. Open the image in a new tab to see the enlarged version. + +
+  Database Restore Overview +
Fig: MySQL Restore Process Overview
+
+ +The restore process consists of the following steps: + +1. At first, a user creates a `MySQL` database where the data will be restored or the user can use the same `MySQL` database. + +2. Then, she creates a `RestoreSession` custom resource that specifies the target database where the backed-up data will be restored, addon information (including restore tasks), the target snapshot to be restored, the `Repository` containing that snapshot, and other additional settings. + +3. KubeStash operator watches for `RestoreSession` custom resources. + +4. When it finds a `RestoreSession` custom resource, it resolves the respective `Addon` and `Function` and prepares a restore `Job` definition. + +5. Then, it creates the `Job` to restore the target. + +6. The `Job` reads necessary information to connect with the database from respective `AppBinding` CR. It also reads backend information and access credentials from `Repository` CR and storage `Secret` respectively. + +7. Then, the `Job` downloads the backed up data from the backend and injects into the desired database. KubeStash pipes the downloaded data to the respective database tool to inject into the database. Hence, restore `Job` does not require a large volume to download entire backup data inside it. + +8. Finally, when the restore process is completed, the `Job` updates the `status.components[*]` field of the `RestoreSession` with restore information of the target database. + +## Next Steps + +- Backup a `MySQL` database using KubeStash by following the guides from [here](/docs/guides/mysql/backup/kubestash/logical/index.md). diff --git a/docs/guides/mysql/backup/stash/_index.md b/docs/guides/mysql/backup/stash/_index.md new file mode 100644 index 0000000000..0136150867 --- /dev/null +++ b/docs/guides/mysql/backup/stash/_index.md @@ -0,0 +1,10 @@ +--- +title: Backup & Restore MySQL +menu: + docs_{{ .version }}: + identifier: guides-mysql-backup-stashv1 + name: Stash + parent: guides-mysql-backup + weight: 40 +menu_name: docs_{{ .version }} +--- \ No newline at end of file diff --git a/docs/guides/mysql/backup/auto-backup/examples/backupblueprint.yaml b/docs/guides/mysql/backup/stash/auto-backup/examples/backupblueprint.yaml similarity index 100% rename from docs/guides/mysql/backup/auto-backup/examples/backupblueprint.yaml rename to docs/guides/mysql/backup/stash/auto-backup/examples/backupblueprint.yaml diff --git a/docs/guides/mysql/backup/auto-backup/examples/sample-mysql-2.yaml b/docs/guides/mysql/backup/stash/auto-backup/examples/sample-mysql-2.yaml similarity index 100% rename from docs/guides/mysql/backup/auto-backup/examples/sample-mysql-2.yaml rename to docs/guides/mysql/backup/stash/auto-backup/examples/sample-mysql-2.yaml diff --git a/docs/guides/mysql/backup/auto-backup/examples/sample-mysql-3.yaml b/docs/guides/mysql/backup/stash/auto-backup/examples/sample-mysql-3.yaml similarity index 100% rename from docs/guides/mysql/backup/auto-backup/examples/sample-mysql-3.yaml rename to docs/guides/mysql/backup/stash/auto-backup/examples/sample-mysql-3.yaml diff --git a/docs/guides/mysql/backup/auto-backup/examples/sample-mysql.yaml b/docs/guides/mysql/backup/stash/auto-backup/examples/sample-mysql.yaml similarity index 100% rename from docs/guides/mysql/backup/auto-backup/examples/sample-mysql.yaml rename to docs/guides/mysql/backup/stash/auto-backup/examples/sample-mysql.yaml diff --git a/docs/guides/mysql/backup/auto-backup/images/sample-mysql-2.png b/docs/guides/mysql/backup/stash/auto-backup/images/sample-mysql-2.png similarity index 100% rename from docs/guides/mysql/backup/auto-backup/images/sample-mysql-2.png rename to docs/guides/mysql/backup/stash/auto-backup/images/sample-mysql-2.png diff --git a/docs/guides/mysql/backup/auto-backup/images/sample-mysql-3.png b/docs/guides/mysql/backup/stash/auto-backup/images/sample-mysql-3.png similarity index 100% rename from docs/guides/mysql/backup/auto-backup/images/sample-mysql-3.png rename to docs/guides/mysql/backup/stash/auto-backup/images/sample-mysql-3.png diff --git a/docs/guides/mysql/backup/auto-backup/images/sample-mysql.png b/docs/guides/mysql/backup/stash/auto-backup/images/sample-mysql.png similarity index 100% rename from docs/guides/mysql/backup/auto-backup/images/sample-mysql.png rename to docs/guides/mysql/backup/stash/auto-backup/images/sample-mysql.png diff --git a/docs/guides/mysql/backup/auto-backup/index.md b/docs/guides/mysql/backup/stash/auto-backup/index.md similarity index 97% rename from docs/guides/mysql/backup/auto-backup/index.md rename to docs/guides/mysql/backup/stash/auto-backup/index.md index 45962d0448..452caad397 100644 --- a/docs/guides/mysql/backup/auto-backup/index.md +++ b/docs/guides/mysql/backup/stash/auto-backup/index.md @@ -3,9 +3,9 @@ title: MySQL Auto-Backup | Stash description: Backup MySQL using Stash Auto-Backup menu: docs_{{ .version }}: - identifier: guides-mysql-backup-auto-backup + identifier: guides-mysql-backup-auto-backup-stashv1 name: Auto-Backup - parent: guides-mysql-backup + parent: guides-mysql-backup-stashv1 weight: 30 menu_name: docs_{{ .version }} section_menu_id: guides @@ -22,7 +22,7 @@ In this tutorial, we are going to show how you can configure a backup blueprint - At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. - Install Stash in your cluster following the steps [here](https://stash.run/docs/latest/setup/install/stash/). - Install KubeDB in your cluster following the steps [here](/docs/setup/README.md). -- If you are not familiar with how Stash backup and restore MySQL databases, please check the following guide [here](/docs/guides/mysql/backup/overview/index.md). +- If you are not familiar with how Stash backup and restore MySQL databases, please check the following guide [here](/docs/guides/mysql/backup/stash/overview/index.md). - If you are not familiar with how auto-backup works in Stash, please check the following guide [here](https://stash.run/docs/latest/guides/auto-backup/overview/). - If you are not familiar with the available auto-backup options for databases in Stash, please check the following guide [here](https://stash.run/docs/latest/guides/auto-backup/database/). @@ -97,7 +97,7 @@ Notice the `prefix` field of `backend` section. We have used some variables in f Let's create the `BackupBlueprint` we have shown above, ```bash -❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/auto-backup/examples/backupblueprint.yaml +❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/auto-backup/examples/backupblueprint.yaml backupblueprint.stash.appscode.com/mysql-backup-template created ``` @@ -153,7 +153,7 @@ Notice the `annotations` section. We are pointing to the `BackupBlueprint` that Let's create the above MySQL CRO, ```bash -❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/auto-backup/examples/sample-mysql.yaml +❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/auto-backup/examples/sample-mysql.yaml mysql.kubedb.com/sample-mysql created ``` @@ -287,7 +287,7 @@ app-sample-mysql-1643879707 BackupConfiguration app-sample-mysql Running Once the backup has been completed successfully, you should see the backed up data has been stored in the bucket at the directory pointed by the `prefix` field of the `Repository`.
- Backup data in GCS Bucket + Backup data in GCS Bucket
Fig: Backup data in GCS Bucket
@@ -339,7 +339,7 @@ Notice the `annotations` section. This time, we have passed a schedule via `stas Let's create the above MySQL CRO, ```bash -❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/auto-backup/examples/sample-mysql-2.yaml +❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/auto-backup/examples/sample-mysql-2.yaml mysql.kubedb.com/sample-mysql-2 created ``` @@ -476,7 +476,7 @@ demo-2 app-sample-mysql-2-1643880964 BackupConfiguration app-sample-mys Once the backup has been completed successfully, you should see that Stash has created a new directory as pointed by the `prefix` field of the new `Repository` and stored the backed up data there.
- Backup data in GCS Bucket + Backup data in GCS Bucket
Fig: Backup data in GCS Bucket
@@ -528,7 +528,7 @@ Notice the `annotations` section. This time, we have passed an argument via `par Let's create the above MySQL CRO, ```bash -❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/auto-backup/examples/sample-mysql-3.yaml +❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/auto-backup/examples/sample-mysql-3.yaml mysql.kubedb.com/sample-mysql-3 created ``` @@ -666,7 +666,7 @@ demo-3 app-sample-mysql-3-1643883304 BackupConfiguration app-sample-mys Once the backup has been completed successfully, you should see that Stash has created a new directory as pointed by the `prefix` field of the new `Repository` and stored the backed up data there.
- Backup data in GCS Bucket + Backup data in GCS Bucket
Fig: Backup data in GCS Bucket
@@ -675,7 +675,7 @@ Once the backup has been completed successfully, you should see that Stash has c To cleanup the resources crated by this tutorial, run the following commands, ```bash -❯ kubectl delete -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/auto-backup/examples/ +❯ kubectl delete -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/auto-backup/examples/ backupblueprint.stash.appscode.com "mysql-backup-template" deleted mysql.kubedb.com "sample-mysql-2" deleted mysql.kubedb.com "sample-mysql-3" deleted diff --git a/docs/guides/mysql/backup/customization/examples/backup/multi-retention-policy.yaml b/docs/guides/mysql/backup/stash/customization/examples/backup/multi-retention-policy.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/backup/multi-retention-policy.yaml rename to docs/guides/mysql/backup/stash/customization/examples/backup/multi-retention-policy.yaml diff --git a/docs/guides/mysql/backup/customization/examples/backup/passing-args.yaml b/docs/guides/mysql/backup/stash/customization/examples/backup/passing-args.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/backup/passing-args.yaml rename to docs/guides/mysql/backup/stash/customization/examples/backup/passing-args.yaml diff --git a/docs/guides/mysql/backup/customization/examples/backup/resource-limit.yaml b/docs/guides/mysql/backup/stash/customization/examples/backup/resource-limit.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/backup/resource-limit.yaml rename to docs/guides/mysql/backup/stash/customization/examples/backup/resource-limit.yaml diff --git a/docs/guides/mysql/backup/customization/examples/backup/specific-user.yaml b/docs/guides/mysql/backup/stash/customization/examples/backup/specific-user.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/backup/specific-user.yaml rename to docs/guides/mysql/backup/stash/customization/examples/backup/specific-user.yaml diff --git a/docs/guides/mysql/backup/customization/examples/repository.yaml b/docs/guides/mysql/backup/stash/customization/examples/repository.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/repository.yaml rename to docs/guides/mysql/backup/stash/customization/examples/repository.yaml diff --git a/docs/guides/mysql/backup/customization/examples/restore/passing-args.yaml b/docs/guides/mysql/backup/stash/customization/examples/restore/passing-args.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/restore/passing-args.yaml rename to docs/guides/mysql/backup/stash/customization/examples/restore/passing-args.yaml diff --git a/docs/guides/mysql/backup/customization/examples/restore/resource-limit.yaml b/docs/guides/mysql/backup/stash/customization/examples/restore/resource-limit.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/restore/resource-limit.yaml rename to docs/guides/mysql/backup/stash/customization/examples/restore/resource-limit.yaml diff --git a/docs/guides/mysql/backup/customization/examples/restore/specific-snapshot.yaml b/docs/guides/mysql/backup/stash/customization/examples/restore/specific-snapshot.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/restore/specific-snapshot.yaml rename to docs/guides/mysql/backup/stash/customization/examples/restore/specific-snapshot.yaml diff --git a/docs/guides/mysql/backup/customization/examples/restore/specific-user.yaml b/docs/guides/mysql/backup/stash/customization/examples/restore/specific-user.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/restore/specific-user.yaml rename to docs/guides/mysql/backup/stash/customization/examples/restore/specific-user.yaml diff --git a/docs/guides/mysql/backup/customization/examples/sample-mysql.yaml b/docs/guides/mysql/backup/stash/customization/examples/sample-mysql.yaml similarity index 100% rename from docs/guides/mysql/backup/customization/examples/sample-mysql.yaml rename to docs/guides/mysql/backup/stash/customization/examples/sample-mysql.yaml diff --git a/docs/guides/mysql/backup/customization/index.md b/docs/guides/mysql/backup/stash/customization/index.md similarity index 98% rename from docs/guides/mysql/backup/customization/index.md rename to docs/guides/mysql/backup/stash/customization/index.md index 227d7105db..6ca5302d00 100644 --- a/docs/guides/mysql/backup/customization/index.md +++ b/docs/guides/mysql/backup/stash/customization/index.md @@ -3,9 +3,9 @@ title: MySQL Backup Customization | Stash description: Customizing MySQL Backup and Restore process with Stash menu: docs_{{ .version }}: - identifier: guides-mysql-backup-customization + identifier: guides-mysql-backup-customization-stashv1 name: Customizing Backup & Restore Process - parent: guides-mysql-backup + parent: guides-mysql-backup-stashv1 weight: 40 menu_name: docs_{{ .version }} section_menu_id: guides diff --git a/docs/guides/mysql/backup/overview/images/backup_overview.svg b/docs/guides/mysql/backup/stash/overview/images/backup_overview.svg similarity index 100% rename from docs/guides/mysql/backup/overview/images/backup_overview.svg rename to docs/guides/mysql/backup/stash/overview/images/backup_overview.svg diff --git a/docs/guides/mysql/backup/overview/images/restore_overview.svg b/docs/guides/mysql/backup/stash/overview/images/restore_overview.svg similarity index 100% rename from docs/guides/mysql/backup/overview/images/restore_overview.svg rename to docs/guides/mysql/backup/stash/overview/images/restore_overview.svg diff --git a/docs/guides/mysql/backup/overview/index.md b/docs/guides/mysql/backup/stash/overview/index.md similarity index 94% rename from docs/guides/mysql/backup/overview/index.md rename to docs/guides/mysql/backup/stash/overview/index.md index 0a7834d053..f1180c2d33 100644 --- a/docs/guides/mysql/backup/overview/index.md +++ b/docs/guides/mysql/backup/stash/overview/index.md @@ -2,9 +2,9 @@ title: Backup & Restore MySQL Overview menu: docs_{{ .version }}: - identifier: guides-mysql-backup-overview + identifier: guides-mysql-backup-overview-stashv1 name: Overview - parent: guides-mysql-backup + parent: guides-mysql-backup-stashv1 weight: 10 menu_name: docs_{{ .version }} section_menu_id: guides @@ -28,7 +28,7 @@ KubeDB uses [Stash](https://stash.run) to backup and restore databases. Stash by The following diagram shows how Stash takes backup of a MySQL database. Open the image in a new tab to see the enlarged version.
-  MySQL Backup Overview +  MySQL Backup Overview
Fig: MySQL Backup Overview
@@ -63,7 +63,7 @@ The backup process consists of the following steps: The following diagram shows how Stash restores backed up data into a MySQL database. Open the image in a new tab to see the enlarged version.
-  Database Restore Overview +  Database Restore Overview
Fig: MySQL Restore Process Overview
@@ -85,4 +85,4 @@ The restore process consists of the following steps: ## Next Steps -- Backup a standalone MySQL server using Stash by following the guides from [here](/docs/guides/mysql/backup/standalone/index.md). +- Backup a standalone MySQL server using Stash by following the guides from [here](/docs/guides/mysql/backup/stash/standalonedalone/index.md). diff --git a/docs/guides/mysql/backup/standalone/examples/backupconfiguration.yaml b/docs/guides/mysql/backup/stash/standalone/examples/backupconfiguration.yaml similarity index 100% rename from docs/guides/mysql/backup/standalone/examples/backupconfiguration.yaml rename to docs/guides/mysql/backup/stash/standalone/examples/backupconfiguration.yaml diff --git a/docs/guides/mysql/backup/standalone/examples/repository.yaml b/docs/guides/mysql/backup/stash/standalone/examples/repository.yaml similarity index 100% rename from docs/guides/mysql/backup/standalone/examples/repository.yaml rename to docs/guides/mysql/backup/stash/standalone/examples/repository.yaml diff --git a/docs/guides/mysql/backup/standalone/examples/restored-mysql.yaml b/docs/guides/mysql/backup/stash/standalone/examples/restored-mysql.yaml similarity index 100% rename from docs/guides/mysql/backup/standalone/examples/restored-mysql.yaml rename to docs/guides/mysql/backup/stash/standalone/examples/restored-mysql.yaml diff --git a/docs/guides/mysql/backup/standalone/examples/restoresession.yaml b/docs/guides/mysql/backup/stash/standalone/examples/restoresession.yaml similarity index 100% rename from docs/guides/mysql/backup/standalone/examples/restoresession.yaml rename to docs/guides/mysql/backup/stash/standalone/examples/restoresession.yaml diff --git a/docs/guides/mysql/backup/standalone/examples/sample-mysql.yaml b/docs/guides/mysql/backup/stash/standalone/examples/sample-mysql.yaml similarity index 100% rename from docs/guides/mysql/backup/standalone/examples/sample-mysql.yaml rename to docs/guides/mysql/backup/stash/standalone/examples/sample-mysql.yaml diff --git a/docs/guides/mysql/backup/standalone/images/sample-mysql-backup.png b/docs/guides/mysql/backup/stash/standalone/images/sample-mysql-backup.png similarity index 100% rename from docs/guides/mysql/backup/standalone/images/sample-mysql-backup.png rename to docs/guides/mysql/backup/stash/standalone/images/sample-mysql-backup.png diff --git a/docs/guides/mysql/backup/standalone/index.md b/docs/guides/mysql/backup/stash/standalone/index.md similarity index 97% rename from docs/guides/mysql/backup/standalone/index.md rename to docs/guides/mysql/backup/stash/standalone/index.md index 1a1fc084cf..36815cf25e 100644 --- a/docs/guides/mysql/backup/standalone/index.md +++ b/docs/guides/mysql/backup/stash/standalone/index.md @@ -3,9 +3,9 @@ title: Backup & Restore MySQL | Stash description: Backup standalone MySQL database using Stash menu: docs_{{ .version }}: - identifier: guides-mysql-backup-standalone + identifier: guides-mysql-backup-standalone-stashv1 name: Standalone MySQL - parent: guides-mysql-backup + parent: guides-mysql-backup-stashv1 weight: 20 menu_name: docs_{{ .version }} section_menu_id: guides @@ -21,7 +21,7 @@ Stash 0.9.0+ supports backup and restoration of MySQL databases. This guide will - Install KubeDB in your cluster following the steps [here](/docs/setup/README.md). - Install Stash in your cluster following the steps [here](https://stash.run/docs/latest/setup/install/stash/). - Install Stash `kubectl` plugin following the steps [here](https://stash.run/docs/latest/setup/install/kubectl-plugin/). -- If you are not familiar with how Stash backup and restore MySQL databases, please check the following guide [here](/docs/guides/mysql/backup/overview/index.md). +- If you are not familiar with how Stash backup and restore MySQL databases, please check the following guide [here](/docs/guides/mysql/backup/stash/overview/index.md). You have to be familiar with following custom resources: @@ -72,7 +72,7 @@ spec: Create the above `MySQL` CRD, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/standalone/examples/sample-mysql.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/standalone/examples/sample-mysql.yaml mysql.kubedb.com/sample-mysql created ``` @@ -295,7 +295,7 @@ spec: Let's create the `Repository` we have shown above, ```bash -$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/standalone/examples/repository.yaml +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/standalone/examples/repository.yaml repository.stash.appscode.com/gcs-repo created ``` @@ -338,7 +338,7 @@ Here, Let's create the `BackupConfiguration` CRD we have shown above, ```bash -$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/standalone/examples/backupconfiguration.yaml +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/standalone/examples/backupconfiguration.yaml backupconfiguration.stash.appscode.com/sample-mysql-backup created ``` @@ -394,7 +394,7 @@ gcs-repo true 6.815 MiB 1 3m39s 30m Now, if we navigate to the GCS bucket, we will see the backed up data has been stored in `demo/mysql/sample-mysql` directory as specified by `.spec.backend.gcs.prefix` field of Repository CRD.
-  Backup data in GCS Bucket +  Backup data in GCS Bucket
Fig: Backup data in GCS Bucket
@@ -462,7 +462,7 @@ spec: Let's create the above database, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/standalone/examples/restored-mysql.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/standalone/examples/restored-mysql.yaml mysql.kubedb.com/restored-mysql created ``` @@ -515,7 +515,7 @@ Here, Let's create the RestoreSession CRD object we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/standalone/examples/restoresession.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/stash/standalone/examples/restoresession.yaml restoresession.stash.appscode.com/sample-mysql-restore created ``` diff --git a/docs/guides/mysql/pitr/archiver.md b/docs/guides/mysql/pitr/archiver.md index ec4d318886..00ad0fae62 100644 --- a/docs/guides/mysql/pitr/archiver.md +++ b/docs/guides/mysql/pitr/archiver.md @@ -446,7 +446,7 @@ $ kubectl delete ns demo ## Next Steps -- Learn about [backup and restore](/docs/guides/mysql/backup/overview/index.md) MySQL database using Stash. +- Learn about [backup and restore](/docs/guides/mysql/backup/stash/overview/index.md) MySQL database using Stash. - Learn about initializing [MySQL with Script](/docs/guides/mysql/initialization/script_source.md). - Learn about [custom MySQLVersions](/docs/guides/mysql/custom-versions/setup.md). - Want to setup MySQL cluster? Check how to [configure Highly Available MySQL Cluster](/docs/guides/mysql/clustering/ha_cluster.md) From e5f76b49bbaf268154d784ed51eba1fcccc7a76b Mon Sep 17 00:00:00 2001 From: Anisur Rahman Date: Wed, 4 Sep 2024 17:40:07 +0600 Subject: [PATCH 02/10] Add logical backup/restore process for PG Signed-off-by: Anisur Rahman --- docs/guides/postgres/README.md | 2 +- .../postgres/backup/kubestash/_index.md | 10 + .../examples/backupconfiguration.yaml | 37 + .../examples/backupstorage.yaml | 17 + .../examples/restoresession.yaml | 21 + .../examples/retentionpolicy.yaml | 15 + .../examples/sample-postgres.yaml | 18 + .../kubestash/application-level/index.md | 754 +++++++++++++++ .../auto-backup/examples/backupstorage.yaml | 17 + .../examples/customize-backupblueprint.yaml | 42 + .../examples/default-backupblueprint.yaml | 39 + .../auto-backup/examples/retentionpolicy.yaml | 15 + .../examples/sample-postgres-2.yaml | 26 + .../auto-backup/examples/sample-postgres.yaml | 21 + .../backup/kubestash/auto-backup/index.md | 863 ++++++++++++++++++ .../backup/multiple-repository.yaml | 42 + .../customization/backup/passing-args.yaml | 38 + .../backup/passing-database.yaml | 39 + .../customization/backup/resources-limit.yaml | 45 + .../customization/backup/specific-user.yaml | 41 + .../customization/common/backupstorage.yaml | 17 + .../customization/common/retentionpolicy.yaml | 15 + .../customization/common/sample-postgres.yaml | 18 + .../backup/kubestash/customization/index.md | 387 ++++++++ .../restore/resources-limit.yaml | 30 + .../restore/specific-snapshot.yaml | 21 + .../customization/restore/specific-user.yaml | 26 + .../logical/examples/backupconfiguration.yaml | 36 + .../logical/examples/backupstorage.yaml | 17 + .../logical/examples/restored-postgres.yaml | 20 + .../logical/examples/restoresession.yaml | 21 + .../logical/examples/retentionpolicy.yaml | 15 + .../logical/examples/sample-postgres.yaml | 18 + .../backup/kubestash/logical/index.md | 760 +++++++++++++++ .../overview/images/backup_overview.svg | 1 + .../overview/images/kubedb_plus_kubestash.svg | 21 + .../overview/images/restore_overview.svg | 1 + .../backup/kubestash/overview/index.md | 98 ++ docs/guides/postgres/backup/stash/_index.md | 10 + .../auto-backup/examples/backupblueprint.yaml | 0 .../auto-backup/examples/sample-pg-1.yaml | 0 .../auto-backup/examples/sample-pg-2.yaml | 0 .../auto-backup/examples/sample-pg-3.yaml | 0 .../auto-backup/images/sample-postgres-1.png | Bin .../auto-backup/images/sample-postgres-2.png | Bin .../auto-backup/images/sample-postgres-3.png | Bin .../backup/{ => stash}/auto-backup/index.md | 22 +- .../backup/multi-retention-policy.yaml | 0 .../examples/backup/resource-limit.yaml | 0 .../backup/specific-database-user.yaml | 0 .../examples/backup/specific-user.yaml | 0 .../customization/examples/repository.yaml | 0 .../examples/restore/passing-args.yaml | 0 .../examples/restore/resource-limit.yaml | 0 .../restore/specific-database-user.yaml | 0 .../examples/restore/specific-snapshot.yaml | 0 .../examples/restore/specific-user.yaml | 0 .../examples/sample-postgres.yaml | 0 .../backup/{ => stash}/customization/index.md | 4 +- .../overview/images/backup_overview.svg | 0 .../overview/images/restore_overview.svg | 0 .../backup/{ => stash}/overview/index.md | 12 +- .../standalone/examples/appbinding.yaml | 0 .../examples/backupconfiguration.yaml | 0 .../standalone/examples/postgres.yaml | 0 .../standalone/examples/repository.yaml | 0 .../examples/restored-postgres.yaml | 0 .../standalone/examples/restoresession.yaml | 0 .../images/sample-postgres-backup.png | Bin .../backup/{ => stash}/standalone/index.md | 18 +- .../configuration/using-config-file.md | 2 +- .../postgres/custom-rbac/using-custom-rbac.md | 2 +- .../postgres/initialization/script_source.md | 2 +- .../monitoring/using-builtin-prometheus.md | 2 +- docs/guides/postgres/pitr/archiver.md | 2 +- .../using-private-registry.md | 2 +- docs/guides/postgres/quickstart/quickstart.md | 2 +- .../postgres/remote-replica/remotereplica.md | 2 +- docs/guides/postgres/synchronous/index.md | 2 +- 79 files changed, 3670 insertions(+), 38 deletions(-) create mode 100644 docs/guides/postgres/backup/kubestash/_index.md create mode 100644 docs/guides/postgres/backup/kubestash/application-level/examples/backupconfiguration.yaml create mode 100644 docs/guides/postgres/backup/kubestash/application-level/examples/backupstorage.yaml create mode 100644 docs/guides/postgres/backup/kubestash/application-level/examples/restoresession.yaml create mode 100644 docs/guides/postgres/backup/kubestash/application-level/examples/retentionpolicy.yaml create mode 100644 docs/guides/postgres/backup/kubestash/application-level/examples/sample-postgres.yaml create mode 100644 docs/guides/postgres/backup/kubestash/application-level/index.md create mode 100644 docs/guides/postgres/backup/kubestash/auto-backup/examples/backupstorage.yaml create mode 100644 docs/guides/postgres/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml create mode 100644 docs/guides/postgres/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml create mode 100644 docs/guides/postgres/backup/kubestash/auto-backup/examples/retentionpolicy.yaml create mode 100644 docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres-2.yaml create mode 100644 docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres.yaml create mode 100644 docs/guides/postgres/backup/kubestash/auto-backup/index.md create mode 100644 docs/guides/postgres/backup/kubestash/customization/backup/multiple-repository.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/backup/passing-args.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/backup/passing-database.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/backup/resources-limit.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/backup/specific-user.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/common/backupstorage.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/common/retentionpolicy.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/common/sample-postgres.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/index.md create mode 100644 docs/guides/postgres/backup/kubestash/customization/restore/resources-limit.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/restore/specific-snapshot.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/restore/specific-user.yaml create mode 100644 docs/guides/postgres/backup/kubestash/logical/examples/backupconfiguration.yaml create mode 100644 docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml create mode 100644 docs/guides/postgres/backup/kubestash/logical/examples/restored-postgres.yaml create mode 100644 docs/guides/postgres/backup/kubestash/logical/examples/restoresession.yaml create mode 100644 docs/guides/postgres/backup/kubestash/logical/examples/retentionpolicy.yaml create mode 100644 docs/guides/postgres/backup/kubestash/logical/examples/sample-postgres.yaml create mode 100644 docs/guides/postgres/backup/kubestash/logical/index.md create mode 100644 docs/guides/postgres/backup/kubestash/overview/images/backup_overview.svg create mode 100644 docs/guides/postgres/backup/kubestash/overview/images/kubedb_plus_kubestash.svg create mode 100644 docs/guides/postgres/backup/kubestash/overview/images/restore_overview.svg create mode 100644 docs/guides/postgres/backup/kubestash/overview/index.md create mode 100644 docs/guides/postgres/backup/stash/_index.md rename docs/guides/postgres/backup/{ => stash}/auto-backup/examples/backupblueprint.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/auto-backup/examples/sample-pg-1.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/auto-backup/examples/sample-pg-2.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/auto-backup/examples/sample-pg-3.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/auto-backup/images/sample-postgres-1.png (100%) rename docs/guides/postgres/backup/{ => stash}/auto-backup/images/sample-postgres-2.png (100%) rename docs/guides/postgres/backup/{ => stash}/auto-backup/images/sample-postgres-3.png (100%) rename docs/guides/postgres/backup/{ => stash}/auto-backup/index.md (96%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/backup/multi-retention-policy.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/backup/resource-limit.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/backup/specific-database-user.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/backup/specific-user.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/repository.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/restore/passing-args.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/restore/resource-limit.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/restore/specific-database-user.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/restore/specific-snapshot.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/restore/specific-user.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/examples/sample-postgres.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/customization/index.md (99%) rename docs/guides/postgres/backup/{ => stash}/overview/images/backup_overview.svg (100%) rename docs/guides/postgres/backup/{ => stash}/overview/images/restore_overview.svg (100%) rename docs/guides/postgres/backup/{ => stash}/overview/index.md (93%) rename docs/guides/postgres/backup/{ => stash}/standalone/examples/appbinding.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/standalone/examples/backupconfiguration.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/standalone/examples/postgres.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/standalone/examples/repository.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/standalone/examples/restored-postgres.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/standalone/examples/restoresession.yaml (100%) rename docs/guides/postgres/backup/{ => stash}/standalone/images/sample-postgres-backup.png (100%) rename docs/guides/postgres/backup/{ => stash}/standalone/index.md (97%) diff --git a/docs/guides/postgres/README.md b/docs/guides/postgres/README.md index 08ef9fe1b3..7cc8130805 100644 --- a/docs/guides/postgres/README.md +++ b/docs/guides/postgres/README.md @@ -46,7 +46,7 @@ aliases: ## User Guide - [Quickstart PostgreSQL](/docs/guides/postgres/quickstart/quickstart.md) with KubeDB Operator. -- How to [Backup & Restore](/docs/guides/postgres/backup/overview/index.md) PostgreSQL database using Stash. +- How to [Backup & Restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL database using Stash. - Initialize [PostgreSQL with Script](/docs/guides/postgres/initialization/script_source.md). - [PostgreSQL Clustering](/docs/guides/postgres/clustering/ha_cluster.md) supported by KubeDB Postgres. - [Streaming Replication](/docs/guides/postgres/clustering/streaming_replication.md) for PostgreSQL clustering. diff --git a/docs/guides/postgres/backup/kubestash/_index.md b/docs/guides/postgres/backup/kubestash/_index.md new file mode 100644 index 0000000000..febbaf5289 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/_index.md @@ -0,0 +1,10 @@ +--- +title: Backup & Restore PostgreSQL | KubeStash +menu: + docs_{{ .version }}: + identifier: guides-pg-backup-stashv2 + name: KubeStash (aka Stash 2.0) + parent: guides-pg-backup + weight: 50 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/postgres/backup/kubestash/application-level/examples/backupconfiguration.yaml b/docs/guides/postgres/backup/kubestash/application-level/examples/backupconfiguration.yaml new file mode 100644 index 0000000000..d2ab57f692 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/application-level/examples/backupconfiguration.yaml @@ -0,0 +1,37 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: manifest-backup + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/application-level/examples/backupstorage.yaml b/docs/guides/postgres/backup/kubestash/application-level/examples/backupstorage.yaml new file mode 100644 index 0000000000..0461b26762 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/application-level/examples/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/application-level/examples/restoresession.yaml b/docs/guides/postgres/backup/kubestash/application-level/examples/restoresession.yaml new file mode 100644 index 0000000000..7ff80b1266 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/application-level/examples/restoresession.yaml @@ -0,0 +1,21 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-postgres + namespace: demo +spec: + manifestOptions: + restoreNamespace: dev + postgres: + db: true + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup-restore + - name: manifest-restore \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/application-level/examples/retentionpolicy.yaml b/docs/guides/postgres/backup/kubestash/application-level/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/application-level/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/application-level/examples/sample-postgres.yaml b/docs/guides/postgres/backup/kubestash/application-level/examples/sample-postgres.yaml new file mode 100644 index 0000000000..4a1abc9f9a --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/application-level/examples/sample-postgres.yaml @@ -0,0 +1,18 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: sample-postgres + namespace: demo +spec: + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/application-level/index.md b/docs/guides/postgres/backup/kubestash/application-level/index.md new file mode 100644 index 0000000000..21a249984f --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/application-level/index.md @@ -0,0 +1,754 @@ +--- +title: Application Level Backup & Restore PostgreSQL | KubeStash +description: Application Level Backup and Restore using KubeStash +menu: + docs_{{ .version }}: + identifier: guides-application-level-backup-stashv2 + name: Application Level Backup + parent: guides-pg-backup-stashv2 + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Application Level Backup and Restore PostgreSQL database using KubeStash + +KubeStash offers application-level backup and restore functionality for `PostgreSQL` databases. It captures both manifest and data backups of any `PostgreSQL` database in a single snapshot. During the restore process, KubeStash first applies the `PostgreSQL` manifest to the cluster and then restores the data into it. + +This guide will give you an overview how you can take application-level backup and restore your `PostgreSQL` databases using `Kubestash`. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. +- Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). +- Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). +- Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- If you are not familiar with how KubeStash backup and restore PostgreSQL databases, please check the following guide [here](/docs/guides/postgres/backup/kubestash/overview/index.md). + +You should be familiar with the following `KubeStash` concepts: + +- [BackupStorage](https://kubestash.com/docs/latest/concepts/crds/backupstorage/) +- [BackupConfiguration](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/) +- [BackupSession](https://kubestash.com/docs/latest/concepts/crds/backupsession/) +- [RestoreSession](https://kubestash.com/docs/latest/concepts/crds/restoresession/) +- [Addon](https://kubestash.com/docs/latest/concepts/crds/addon/) +- [Function](https://kubestash.com/docs/latest/concepts/crds/function/) +- [Task](https://kubestash.com/docs/latest/concepts/crds/addon/#task-specification) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/guides/postgres/backup/kubestash/application-level/examples](https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/application-level/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + +## Backup PostgreSQL + +KubeStash supports backups for `PostgreSQL` instances across different configurations, including Standalone and HA Cluster setups. In this demonstration, we'll focus on a `PostgreSQL` database using HA cluster configuration. The backup and restore process is similar for Standalone configuration. + +This section will demonstrate how to take application-level backup of a `PostgreSQL` database. Here, we are going to deploy a `PostgreSQL` database using KubeDB. Then, we are going to back up the database at the application level to a `GCS` bucket. Finally, we will restore the entire `PostgreSQL` database. + +### Deploy Sample PostgreSQL Database + +Let's deploy a sample `PostgreSQL` database and insert some data into it. + +**Create PostgreSQL CR:** + +Below is the YAML of a sample `PostgreSQL` CR that we are going to create for this tutorial: + +```yaml +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: sample-postgres + namespace: demo +spec: + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +Create the above `PostgreSQL` CR, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/application-level/examples/sample-postgres.yaml +postgres.kubedb.com/sample-postgres created +``` + +KubeDB will deploy a `PostgreSQL` database according to the above specification. It will also create the necessary `Secrets` and `Services` to access the database. + +Let's check if the database is ready to use, + +```bash +$ kubectl get pg -n demo sample-postgres +NAME VERSION STATUS AGE +sample-postgres 16.1 Ready 5m1s +``` + +The database is `Ready`. Verify that KubeDB has created a `Secret` and a `Service` for this database using the following commands, + +```bash +$ kubectl get secret -n demo +NAME TYPE DATA AGE +sample-postgres-auth kubernetes.io/basic-auth 2 5m20s + +$ kubectl get service -n demo -l=app.kubernetes.io/instance=sample-postgres +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +sample-postgres ClusterIP 10.96.23.177 5432/TCP,2379/TCP 5m55s +sample-postgres-pods ClusterIP None 5432/TCP,2380/TCP,2379/TCP 5m55s +sample-postgres-standby ClusterIP 10.96.26.118 5432/TCP 5m55s +``` + +Here, we have to use service `sample-postgres` and secret `sample-postgres-auth` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/postgres/concepts/appbinding.md) CR that holds the necessary information to connect with the database. + + +**Verify AppBinding:** + +Verify that the `AppBinding` has been created successfully using the following command, + +```bash +$ kubectl get appbindings -n demo +NAME TYPE VERSION AGE +sample-postgres kubedb.com/postgres 16.1 9m30s +``` + +Let's check the YAML of the above `AppBinding`, + +```bash +$ kubectl get appbindings -n demo sample-postgres -o yaml +``` + +```yaml +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"kubedb.com/v1","kind":"Postgres","metadata":{"annotations":{},"name":"sample-postgres","namespace":"demo"},"spec":{"deletionPolicy":"DoNotTerminate","replicas":3,"standbyMode":"Hot","storage":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"1Gi"}}},"storageType":"Durable","streamingMode":"Synchronous","version":"16.1"}} + creationTimestamp: "2024-09-04T10:07:04Z" + generation: 1 + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: sample-postgres + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: postgreses.kubedb.com + name: sample-postgres + namespace: demo + ownerReferences: + - apiVersion: kubedb.com/v1 + blockOwnerDeletion: true + controller: true + kind: Postgres + name: sample-postgres + uid: 0810a96c-a2b6-4e8a-a70a-51753660450c + resourceVersion: "245972" + uid: 73bdba85-c932-464b-93a8-7f1ba8dfff1b +spec: + appRef: + apiGroup: kubedb.com + kind: Postgres + name: sample-postgres + namespace: demo + clientConfig: + service: + name: sample-postgres + path: / + port: 5432 + query: sslmode=disable + scheme: postgresql + parameters: + apiVersion: appcatalog.appscode.com/v1alpha1 + kind: StashAddon + stash: + addon: + backupTask: + name: postgres-backup-16.1 + restoreTask: + name: postgres-restore-16.1 + secret: + name: sample-postgres-auth + type: kubedb.com/postgres + version: "16.1" +``` + +KubeStash uses the `AppBinding` CR to connect with the target database. It requires the following two fields to set in AppBinding's `.spec` section. + +Here, + +- `.spec.clientConfig.service.name` specifies the name of the Service that connects to the database. +- `.spec.secret` specifies the name of the Secret that holds necessary credentials to access the database. +- `.spec.type` specifies the types of the app that this AppBinding is pointing to. KubeDB generated AppBinding follows the following format: `/`. + +**Insert Sample Data:** + +Now, we are going to exec into one of the database pod and create some sample data. At first, find out the database `Pod` using the following command, + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-postgres" +NAME READY STATUS RESTARTS AGE +sample-postgres-0 2/2 Running 0 16m +sample-postgres-1 2/2 Running 0 13m +sample-postgres-2 2/2 Running 0 13m +``` + +Now, let’s exec into the pod and create a table, + +```bash +$ kubectl exec -it -n demo sample-postgres-0 -- sh + +# login as "postgres" superuser. +/ $ psql -U postgres +psql (16.1) +Type "help" for help. + +# list available databases +postgres=# \l + List of databases + Name | Owner | Encoding | Locale Provider | Collate | Ctype | ICU Locale | ICU Rules | Access privileges +---------------+----------+----------+-----------------+------------+------------+------------+-----------+----------------------- + kubedb_system | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + postgres | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + template0 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres + template1 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres +(4 rows) + +# create a database named "demo" +postgres=# create database demo; +CREATE DATABASE + +# verify that the "demo" database has been created +postgres=# \l + List of databases + Name | Owner | Encoding | Locale Provider | Collate | Ctype | ICU Locale | ICU Rules | Access privileges +---------------+----------+----------+-----------------+------------+------------+------------+-----------+----------------------- + demo | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + kubedb_system | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + postgres | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + template0 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres + template1 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres +(5 rows) + +# connect to the "demo" database +postgres=# \c demo +You are now connected to database "demo" as user "postgres". + +# create a sample table +demo=# CREATE TABLE COMPANY( NAME TEXT NOT NULL, EMPLOYEE INT NOT NULL); +CREATE TABLE + +# verify that the table has been created +demo=# \d + List of relations + Schema | Name | Type | Owner +--------+---------+-------+---------- + public | company | table | postgres +(1 row) + +# quit from the database +demo=# \q + +# exit from the pod +/ $ exit +``` + +Now, we are ready to backup the database. + +### Prepare Backend + +We are going to store our backed up data into a `GCS` bucket. We have to create a `Secret` with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). + +**Create Secret:** + +Let's create a secret called `gcs-secret` with access credentials to our desired GCS bucket, + +```bash +$ echo -n '' > GOOGLE_PROJECT_ID +$ cat /path/to/downloaded-sa-key.json > GOOGLE_SERVICE_ACCOUNT_JSON_KEY +$ kubectl create secret generic -n demo gcs-secret \ + --from-file=./GOOGLE_PROJECT_ID \ + --from-file=./GOOGLE_SERVICE_ACCOUNT_JSON_KEY +secret/gcs-secret created +``` + +**Create BackupStorage:** + +Now, create a `BackupStorage` using this secret. Below is the YAML of `BackupStorage` CR we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: WipeOut +``` + +Let's create the BackupStorage we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml +backupstorage.storage.kubestash.com/gcs-storage created +``` + +Now, we are ready to backup our database to our desired backend. + +**Create RetentionPolicy:** + +Now, let's create a `RetentionPolicy` to specify how the old Snapshots should be cleaned up. + +Below is the YAML of the `RetentionPolicy` object that we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Let’s create the above `RetentionPolicy`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples/retentionpolicy.yaml +retentionpolicy.storage.kubestash.com/demo-retention created +``` + +### Backup + +We have to create a `BackupConfiguration` targeting respective `sample-postgres` PostgreSQL database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. + +At first, we need to create a secret with a Restic password for backup data encryption. + +**Create Secret:** + +Let's create a secret called `encrypt-secret` with the Restic password, + +```bash +$ echo -n 'changeit' > RESTIC_PASSWORD +$ kubectl create secret generic -n demo encrypt-secret \ + --from-file=./RESTIC_PASSWORD \ +secret "encrypt-secret" created +``` + +**Create BackupConfiguration:** + +Below is the YAML for `BackupConfiguration` CR to take application-level backup of the `sample-postgres` database that we have deployed earlier, + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: manifest-backup + - name: logical-backup +``` + +- `.spec.sessions[*].schedule` specifies that we want to backup at `5 minutes` interval. +- `.spec.target` refers to the targeted `sample-postgres` PostgreSQL database that we created earlier. +- `.spec.sessions[*].addon.tasks[*].name[*]` specifies that both the `manifest-backup` and `logical-backup` will be taken together. + + +Let's create the `BackupConfiguration` CR that we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/kubestash/application-level/examples/backupconfiguration.yaml +backupconfiguration.core.kubestash.com/sample-postgres-backup created +``` + +**Verify Backup Setup Successful** + +If everything goes well, the phase of the `BackupConfiguration` should be `Ready`. The `Ready` phase indicates that the backup setup is successful. Let's verify the `Phase` of the BackupConfiguration, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +sample-postgres-backup Ready 2m50s +``` + +Additionally, we can verify that the `Repository` specified in the `BackupConfiguration` has been created using the following command, + +```bash +$ kubectl get repo -n demo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-postgres-repo 0 0 B Ready 3m +``` + +KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/postgres` directory. + +**Verify CronJob:** + +It will also create a `CronJob` with the schedule specified in `spec.sessions[*].scheduler.schedule` field of `BackupConfiguration` CR. + +Verify that the `CronJob` has been created using the following command, + +```bash +$ kubectl get cronjob -n demo +NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE +trigger-sample-postgres-backup-frequent-backup */5 * * * * 0 2m45s 3m25s +``` + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +sample-postgres-backup-frequent-backup-1725449400 BackupConfiguration sample-postgres-backup Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-postgres-backup` has been updated by the following command, + +```bash +$ kubectl get repository -n demo sample-postgres-backup +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +sample-postgres-backup true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=gcs-postgres-repo +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725449400 gcs-postgres-repo frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubedb.com/db-version: ` +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725449400 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-09-05T09:08:03Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubedb.com/db-version: "16.1" + kubestash.com/app-ref-kind: Postgres + kubestash.com/app-ref-name: sample-postgres + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: gcs-postgres-repo + name: gcs-postgres-repo-sample-postgreckup-frequent-backup-1725449400 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: gcs-postgres-repo + uid: fa9086e5-285a-4b4a-9096-072bf7dbe2f7 + resourceVersion: "289843" + uid: 43f17a3f-4ac7-443c-a139-151f2e5bf462 +spec: + appRef: + apiGroup: kubedb.com + kind: Postgres + name: sample-postgres + namespace: demo + backupSession: sample-postgres-backup-frequent-backup-1725527283 + deletionPolicy: Delete + repository: gcs-postgres-repo + session: frequent-backup + snapshotID: 01J70Q1NT6FW11YBBARRFJ6SYB + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 6.684476865s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: 4b820700710f9f7b6a8d5b052367b51875e68dcd9052c749a686506db6a66374 + size: 3.345 KiB + uploaded: 3.634 KiB + size: 1.135 KiB + manifest: + driver: Restic + duration: 7.477728298s + integrity: true + path: repository/v1/frequent-backup/manifest + phase: Succeeded + resticStats: + - hostPath: /kubestash-tmp/manifest + id: 9da4d1b7df6dd946e15a8a0d2a2a3c14776351e27926156770530ca03f6f8002 + size: 3.064 KiB + uploaded: 1.443 KiB + size: 2.972 KiB + conditions: + - lastTransitionTime: "2024-09-05T09:08:03Z" + message: Recent snapshot list updated successfully + reason: SuccessfullyUpdatedRecentSnapshotList + status: "True" + type: RecentSnapshotListUpdated + - lastTransitionTime: "2024-09-05T09:08:49Z" + message: Metadata uploaded to backend successfully + reason: SuccessfullyUploadedSnapshotMetadata + status: "True" + type: SnapshotMetadataUploaded + integrity: true + phase: Succeeded + size: 4.106 KiB + snapshotTime: "2024-09-05T09:08:03Z" + totalComponents: 2 +``` + +> KubeStash uses the `pg_dump` command to take backups of target PostgreSQL databases. Therefore, the component name for `logical backups` is set as `dump`. + +> KubeStash set component name as `manifest` for the `manifest backup` of PostgreSQL databases. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/popstgres/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/postgres/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Restore + +In this section, we are going to restore the entire database from the backup that we have taken in the previous section. + +For this tutorial, we will restore the database in a separate namespace called `dev`. + +First, create the namespace by running the following command: + +```bash +$ kubectl create ns dev +namespace/dev created +``` + +#### Create RestoreSession: + +We need to create a RestoreSession CR. + +Below, is the contents of YAML file of the `RestoreSession` CR that we are going to create to restore the entire database. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-postgres + namespace: demo +spec: + manifestOptions: + restoreNamespace: dev + postgres: + db: true + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup-restore + - name: manifest-restore +``` + +Here, + +- `.spec.manifestOptions.postgres.db` specifies whether to restore the DB manifest or not. +- `.spec.dataSource.repository` specifies the Repository object that holds the backed up data. +- `.spec.dataSource.snapshot` specifies to restore from latest `Snapshot`. +- `.spec.addon.tasks[*]` specifies that both the `manifest-restore` and `logical-backup-restore` tasks. + +Let's create the RestoreSession CR object we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/application-level/examples/restoresession.yaml +restoresession.core.kubestash.com/restore-sample-postgres created +``` + +Once, you have created the `RestoreSession` object, KubeStash will create restore Job. Run the following command to watch the phase of the `RestoreSession` object, + +```bash +$ watch kubectl get restoresession -n demo +Every 2.0s: kubectl get restores... AppsCode-PC-03: Wed Aug 21 10:44:05 2024 +NAME REPOSITORY FAILURE-POLICY PHASE DURATION AGE +restore-sample-postgres gcs-postgres-repo Succeeded 3s 53s +``` + +The `Succeeded` phase means that the restore process has been completed successfully. + + +#### Verify Restored PostgreSQL Manifest: + +In this section, we will verify whether the desired `PostgreSQL` database manifest has been successfully applied to the cluster. + +```bash +$ kubectl get postgres -n dev +NAME VERSION STATUS AGE +sample-postgres 16.1 Ready 9m46s +``` + +The output confirms that the `PostgreSQL` database has been successfully created with the same configuration as it had at the time of backup. + + +#### Verify Restored Data: + +In this section, we are going to verify whether the desired data has been restored successfully. We are going to connect to the database server and check whether the database and the table we created earlier in the original database are restored. + +At first, check if the database has gone into **`Ready`** state by the following command, + +```bash +$ kubectl get postgres -n dev sample-postgres +NAME VERSION STATUS AGE +sample-postgres 16.1 Ready 9m46s +``` + +Now, find out the database `Pod` by the following command, + +```bash +$ kubectl get pods -n dev --selector="app.kubernetes.io/instance=sample-postgres" +NAME READY STATUS RESTARTS AGE +sample-postgres-0 2/2 Running 0 12m +sample-postgres-1 2/2 Running 0 12m +sample-postgres-2 2/2 Running 0 12m +``` + + +Now, lets exec one of the Pod and verify restored data. + +```bash +$ kubectl exec -it -n dev sample-postgres-0 -- /bin/sh +# login as "postgres" superuser. +/ # psql -U postgres +psql (11.11) +Type "help" for help. + +# verify that the "demo" database has been restored +postgres=# \l + List of databases + Name | Owner | Encoding | Locale Provider | Collate | Ctype | ICU Locale | ICU Rules | Access privileges +---------------+----------+----------+-----------------+------------+------------+------------+-----------+----------------------- + demo | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + kubedb_system | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + postgres | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + template0 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres + template1 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres +(5 rows) + +# connect to the "demo" database +postgres=# \c demo +You are now connected to database "demo" as user "postgres". + +# verify that the sample table has been restored +demo=# \d + List of relations + Schema | Name | Type | Owner +--------+---------+-------+---------- + public | company | table | postgres +(1 row) + +# disconnect from the database +demo=# \q + +# exit from the pod +/ # exit +``` + +So, from the above output, we can see the `demo` database we had created in the original database `sample-postgres` has been restored successfully. + +## Cleanup + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl delete backupconfigurations.core.kubestash.com -n demo sample-postgres-backup +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention +kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-postgres +kubectl delete backupstorage -n demo gcs-storage +kubectl delete secret -n demo gcs-secret +kubectl delete secret -n demo encrypt-secret +kubectl delete postgres -n demo sample-postgres +kubectl delete postgres -n dev sample-postgres +``` diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/examples/backupstorage.yaml b/docs/guides/postgres/backup/kubestash/auto-backup/examples/backupstorage.yaml new file mode 100644 index 0000000000..0461b26762 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/auto-backup/examples/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml b/docs/guides/postgres/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml new file mode 100644 index 0000000000..ea1a2eceda --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml @@ -0,0 +1,42 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: postgres-customize-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + # ============== Blueprint for Backends of BackupConfiguration ================= + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + # ============== Blueprint for Sessions of BackupConfiguration ================= + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: ${schedule} + jobTemplate: + backoffLimit: 1 + repositories: + - name: ${repoName} + backend: gcs-backend + directory: ${namespace}/${targetName} + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup + params: + backupCmd: pg_dump + args: ${targetedDatabase} \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml b/docs/guides/postgres/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml new file mode 100644 index 0000000000..627cdfe5a0 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml @@ -0,0 +1,39 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: postgres-default-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + # ============== Blueprint for Backends of BackupConfiguration ================= + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + # ============== Blueprint for Sessions of BackupConfiguration ================= + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: default-blueprint + backend: gcs-backend + directory: /default-blueprint + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/examples/retentionpolicy.yaml b/docs/guides/postgres/backup/kubestash/auto-backup/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/auto-backup/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres-2.yaml b/docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres-2.yaml new file mode 100644 index 0000000000..78e03bb22c --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres-2.yaml @@ -0,0 +1,26 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: sample-postgres-2 + namespace: demo + annotations: + blueprint.kubestash.com/name: postgres-customize-backup-blueprint + blueprint.kubestash.com/namespace: demo + variables.kubestash.com/schedule: "*/10 * * * *" + variables.kubestash.com/repoName: customize-blueprint + variables.kubestash.com/namespace: demo + variables.kubestash.com/targetName: sample-postgres-2 + variables.kubestash.com/targetedDatabase: postgres +spec: + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres.yaml b/docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres.yaml new file mode 100644 index 0000000000..6783528d0f --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres.yaml @@ -0,0 +1,21 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: sample-postgres + namespace: demo + annotations: + blueprint.kubestash.com/name: postgres-default-backup-blueprint + blueprint.kubestash.com/namespace: demo +spec: + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/index.md b/docs/guides/postgres/backup/kubestash/auto-backup/index.md new file mode 100644 index 0000000000..a93e780858 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/auto-backup/index.md @@ -0,0 +1,863 @@ +--- +title: PostgreSQL Auto-Backup | KubeStash +description: Backup PostgreSQL using KubeStash Auto-Backup +menu: + docs_{{ .version }}: + identifier: guides-pg-auto-backup-stashv2 + name: Auto-Backup + parent: guides-pg-backup-stashv2 + weight: 30 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Backup PostgreSQL using KubeStash Auto-Backup + +KubeStash can automatically be configured to backup any `PostgreSQL` databases in your cluster. KubeStash enables cluster administrators to deploy backup `blueprints` ahead of time so database owners can easily backup any `PostgreSQL` database with a few annotations. + +In this tutorial, we are going to show how you can configure a backup blueprint for `PostgreSQL` databases in your cluster and backup them with a few annotations. + + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. +- Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). +- Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). +- Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- If you are not familiar with how KubeStash backup and restore PostgreSQL databases, please check the following guide [here](/docs/guides/postgres/backup/kubestash/overview/index.md). + +You should be familiar with the following `KubeStash` concepts: + +- [BackupStorage](https://kubestash.com/docs/latest/concepts/crds/backupstorage/) +- [BackupConfiguration](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/) +- [BackupSession](https://kubestash.com/docs/latest/concepts/crds/backupsession/) +- [RestoreSession](https://kubestash.com/docs/latest/concepts/crds/restoresession/) +- [Addon](https://kubestash.com/docs/latest/concepts/crds/addon/) +- [Function](https://kubestash.com/docs/latest/concepts/crds/function/) +- [Task](https://kubestash.com/docs/latest/concepts/crds/addon/#task-specification) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/guides/postgres/backup/kubestash/auto-backup/examples](https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/auto-backup/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + + +### Prepare Backend + +We are going to store our backed up data into a `GCS` bucket. We have to create a `Secret` with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). + +**Create Secret:** + +Let's create a secret called `gcs-secret` with access credentials to our desired GCS bucket, + +```bash +$ echo -n '' > GOOGLE_PROJECT_ID +$ cat /path/to/downloaded-sa-key.json > GOOGLE_SERVICE_ACCOUNT_JSON_KEY +$ kubectl create secret generic -n demo gcs-secret \ + --from-file=./GOOGLE_PROJECT_ID \ + --from-file=./GOOGLE_SERVICE_ACCOUNT_JSON_KEY +secret/gcs-secret created +``` + +**Create BackupStorage:** + +Now, create a `BackupStorage` using this secret. Below is the YAML of `BackupStorage` CR we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: blueprint + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: WipeOut +``` + +Let's create the BackupStorage we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/auto-backup/examples/backupstorage.yaml +backupstorage.storage.kubestash.com/gcs-storage created +``` + +Now, we are ready to backup our database to our desired backend. + +**Create RetentionPolicy:** + +Now, let's create a `RetentionPolicy` to specify how the old Snapshots should be cleaned up. + +Below is the YAML of the `RetentionPolicy` object that we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Let’s create the above `RetentionPolicy`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/auto-backup/examples/retentionpolicy.yaml +retentionpolicy.storage.kubestash.com/demo-retention created +``` + +**Create Secret:** + +We also need to create a secret with a `Restic` password for backup data encryption. + +Let's create a secret called `encrypt-secret` with the Restic password, + +```bash +$ echo -n 'changeit' > RESTIC_PASSWORD +$ kubectl create secret generic -n demo encrypt-secret \ + --from-file=./RESTIC_PASSWORD \ +secret "encrypt-secret" created +``` + +## Auto-backup with default configurations + +In this section, we are going to backup a `PostgreSQL` database of `demo` namespace. We are going to use the default configurations which will be specified in the `BackupBlueprint` CR. + +**Prepare Backup Blueprint** + +A `BackupBlueprint` allows you to specify a template for the `Repository`,`Session` or `Variables` of `BackupConfiguration` in a Kubernetes native way. + +Now, we have to create a `BackupBlueprint` CR with a blueprint for `BackupConfiguration` object. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: postgres-default-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + # ============== Blueprint for Backends of BackupConfiguration ================= + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + # ============== Blueprint for Sessions of BackupConfiguration ================= + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: default-blueprint + backend: gcs-backend + directory: /default-blueprint + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup +``` + +Here, + +- `.spec.backupConfigurationTemplate.backends[*].storageRef` refers our earlier created `gcs-storage` backupStorage. +- `.spec.backupConfigurationTemplate.sessions[*].schedule` specifies that we want to backup the database at `5 minutes` interval. + +Let's create the `BackupBlueprint` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml +backupblueprint.core.kubestash.com/postgres-default-backup-blueprint created +``` + +Now, we are ready to backup our `PostgreSQL` databases using few annotations. You can check available auto-backup annotations for a databases from [here](https://kubestash.com/docs/latest/concepts/crds/backupblueprint/). + +**Create Database** + +Now, we are going to create an `PostgreSQL` CR in demo namespace. + +Below is the YAML of the `PostgreSQL` object that we are going to create, + +```yaml +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: sample-postgres + namespace: demo + annotations: + blueprint.kubestash.com/name: postgres-default-backup-blueprint + blueprint.kubestash.com/namespace: demo +spec: + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +Here, + +- `.spec.annotations.blueprint.kubestash.com/name: postgres-default-backup-blueprint` specifies the name of the `BackupBlueprint` that will use in backup. +- `.spec.annotations.blueprint.kubestash.com/namespace: demo` specifies the name of the `namespace` where the `BackupBlueprint` resides. + +Let's create the `PostgreSQL` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres.yaml +postgres.kubedb.com/sample-postgres created +``` + +**Verify BackupConfiguration** + +If everything goes well, KubeStash should create a `BackupConfiguration` for our PostgreSQL in demo namespace and the phase of that `BackupConfiguration` should be `Ready`. Verify the `BackupConfiguration` object by the following command, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +appbinding-sample-postgres Ready 2m50m +``` + +Now, let’s check the YAML of the `BackupConfiguration`. + +```bash +$ kubectl get backupconfiguration -n demo appbinding-sample-postgres -o yaml +``` + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + creationTimestamp: "2024-09-05T10:53:48Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + app.kubernetes.io/managed-by: kubestash.com + kubestash.com/invoker-name: postgres-default-backup-blueprint + kubestash.com/invoker-namespace: demo + name: appbinding-sample-postgres + namespace: demo + resourceVersion: "298502" + uid: b6537c60-051f-4348-9ca4-c28f3880dbc1 +spec: + backends: + - name: gcs-backend + retentionPolicy: + name: demo-retention + namespace: demo + storageRef: + name: gcs-storage + namespace: demo + sessions: + - addon: + name: postgres-addon + tasks: + - name: logical-backup + name: frequent-backup + repositories: + - backend: gcs-backend + directory: /default-blueprint + encryptionSecret: + name: encrypt-secret + namespace: demo + name: default-blueprint + scheduler: + jobTemplate: + backoffLimit: 1 + template: + controller: {} + metadata: {} + spec: + resources: {} + schedule: '*/5 * * * *' + sessionHistoryLimit: 3 + target: + apiGroup: kubedb.com + kind: Postgres + name: sample-postgres + namespace: demo +status: + backends: + - name: gcs-backend + ready: true + retentionPolicy: + found: true + ref: + name: demo-retention + namespace: demo + storage: + phase: Ready + ref: + name: gcs-storage + namespace: demo + conditions: + - lastTransitionTime: "2024-09-05T10:53:48Z" + message: Validation has been passed successfully. + reason: ResourceValidationPassed + status: "True" + type: ValidationPassed + dependencies: + - found: true + kind: Addon + name: postgres-addon + phase: Ready + repositories: + - name: default-blueprint + phase: Ready + sessions: + - conditions: + - lastTransitionTime: "2024-09-05T10:53:59Z" + message: Scheduler has been ensured successfully. + reason: SchedulerEnsured + status: "True" + type: SchedulerEnsured + - lastTransitionTime: "2024-09-05T10:53:59Z" + message: Initial backup has been triggered successfully. + reason: SuccessfullyTriggeredInitialBackup + status: "True" + type: InitialBackupTriggered + name: frequent-backup + targetFound: true +``` + +Notice the `spec.backends`, `spec.sessions` and `spec.target` sections, KubeStash automatically resolved those info from the `BackupBluePrint` and created above `BackupConfiguration`. + + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +appbinding-sample-postgres-frequent-backup-1725533628 BackupConfiguration appbinding-sample-postgres Succeeded 23s 6m40s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-postgres-backup` has been updated by the following command, + +```bash +$ kubectl get repository -n demo default-blueprint +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +default-blueprint true 3 1.559 KiB Ready 80s 7m32s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=default-blueprint +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +default-blueprint-appbinding-samgres-frequent-backup-1725533628 default-blueprint frequent-backup 2024-09-05T10:53:59Z Delete Succeeded 7m48s +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubedb.com/db-version: ` +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo default-blueprint-appbinding-samgres-frequent-backup-1725533628 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-09-05T10:53:59Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubedb.com/db-version: "16.1" + kubestash.com/app-ref-kind: Postgres + kubestash.com/app-ref-name: sample-postgres + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: default-blueprint + name: default-blueprint-appbinding-samgres-frequent-backup-1725533628 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: default-blueprint + uid: 1125a82f-2bd8-4029-aae6-078ff5413383 + resourceVersion: "298559" + uid: c179b758-6ba4-4a32-81f1-fa41ba3dc527 +spec: + appRef: + apiGroup: kubedb.com + kind: Postgres + name: sample-postgres + namespace: demo + backupSession: appbinding-sample-postgres-frequent-backup-1725533628 + deletionPolicy: Delete + repository: default-blueprint + session: frequent-backup + snapshotID: 01J70X3MGSYT4TJK77R8YXEV3T + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 5.952466363s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: a30f8ec138e24cbdbcce088a73e5b9d73a58750c38793ef05ff7d570148ddd2c + size: 3.345 KiB + uploaded: 3.637 KiB + size: 1.132 KiB + conditions: + - lastTransitionTime: "2024-09-05T10:53:59Z" + message: Recent snapshot list updated successfully + reason: SuccessfullyUpdatedRecentSnapshotList + status: "True" + type: RecentSnapshotListUpdated + - lastTransitionTime: "2024-09-05T10:54:20Z" + message: Metadata uploaded to backend successfully + reason: SuccessfullyUploadedSnapshotMetadata + status: "True" + type: SnapshotMetadataUploaded + integrity: true + phase: Succeeded + size: 1.132 KiB + snapshotTime: "2024-09-05T10:53:59Z" + totalComponents: 1 +``` + +> KubeStash uses the `pg_dump` command to take backups of target PostgreSQL databases. Therefore, the component name for `logical backups` is set as `dump`. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `blueprint/default-blueprint/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/default-blueprint/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + + +## Auto-backup with custom configurations + +In this section, we are going to backup a `PostgreSQL` database of `demo` namespace. We are going to use the custom configurations which will be specified in the `BackupBlueprint` CR. + +**Prepare Backup Blueprint** + +A `BackupBlueprint` allows you to specify a template for the `Repository`,`Session` or `Variables` of `BackupConfiguration` in a Kubernetes native way. + +Now, we have to create a `BackupBlueprint` CR with a blueprint for `BackupConfiguration` object. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: postgres-customize-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + # ============== Blueprint for Backends of BackupConfiguration ================= + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + # ============== Blueprint for Sessions of BackupConfiguration ================= + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: ${schedule} + jobTemplate: + backoffLimit: 1 + repositories: + - name: ${repoName} + backend: gcs-backend + directory: ${namespace}/${targetName} + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup + params: + backupCmd: pg_dump + args: ${targetedDatabase} +``` + +Note that we have used some variables (format: `${}`) in different fields. KubeStash will substitute these variables with values from the respective target’s annotations. You’re free to use any variables you like. + +Here, + +- `.spec.backupConfigurationTemplate.backends[*].storageRef` refers our earlier created `gcs-storage` backupStorage. +- `.spec.backupConfigurationTemplate.sessions[*]`: + - `.schedule` defines `${schedule}` variable, which determines the time interval for the backup. + - `.repositories[*].name` defines the `${repoName}` variable, which specifies the name of the backup `Repository`. + - `.repositories[*].directory` defines two variables, `${namespace}` and `${targetName}`, which are used to determine the path where the backup will be stored. + - `.addon.tasks[*].params.args` defines `${targetedDatabase}` variable, which identifies a single database to backup. + +Let's create the `BackupBlueprint` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml +backupblueprint.core.kubestash.com/postgres-customize-backup-blueprint created +``` + +Now, we are ready to backup our `PostgreSQL` databases using few annotations. You can check available auto-backup annotations for a databases from [here](https://kubestash.com/docs/latest/concepts/crds/backupblueprint/). + + +**Create Database** + +Now, we are going to create an `PostgreSQL` CR in demo namespace. + +Below is the YAML of the `PostgreSQL` object that we are going to create, + +```yaml +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: sample-postgres-2 + namespace: demo + annotations: + blueprint.kubestash.com/name: postgres-customize-backup-blueprint + blueprint.kubestash.com/namespace: demo + variables.kubestash.com/schedule: "*/10 * * * *" + variables.kubestash.com/repoName: customize-blueprint + variables.kubestash.com/namespace: demo + variables.kubestash.com/targetName: sample-postgres-2 + variables.kubestash.com/targetedDatabase: postgres +spec: + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +Notice the `metadata.annotations` field, where we have defined the annotations related to the automatic backup configuration. Specifically, we've set the `BackupBlueprint` name as `postgres-customize-backup-blueprint` and the namespace as `demo`. We have also provided values for the blueprint template variables, such as the backup `schedule`, `repositoryName`, `namespace`, `targetName`, and `targetedDatabase`. These annotations will be used to create a `BackupConfiguration` for this `postgreSQL` database. + +Let's create the `PostgreSQL` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/auto-backup/examples/sample-postgres-2.yaml +postgres.kubedb.com/sample-postgres-2 created +``` + +**Verify BackupConfiguration** + +If everything goes well, KubeStash should create a `BackupConfiguration` for our PostgreSQL in demo namespace and the phase of that `BackupConfiguration` should be `Ready`. Verify the `BackupConfiguration` object by the following command, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +appbinding-sample-postgres-2 Ready 2m50m +``` + +Now, let’s check the YAML of the `BackupConfiguration`. + +```bash +$ kubectl get backupconfiguration -n demo appbinding-sample-postgres-2 -o yaml +``` + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + creationTimestamp: "2024-09-05T12:39:37Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + app.kubernetes.io/managed-by: kubestash.com + kubestash.com/invoker-name: postgres-customize-backup-blueprint + kubestash.com/invoker-namespace: demo + name: appbinding-sample-postgres-2 + namespace: demo + resourceVersion: "309511" + uid: b4091166-2813-4183-acda-e2c80eaedbb5 +spec: + backends: + - name: gcs-backend + retentionPolicy: + name: demo-retention + namespace: demo + storageRef: + name: gcs-storage + namespace: demo + sessions: + - addon: + name: postgres-addon + tasks: + - name: logical-backup + params: + args: postgres + backupCmd: pg_dump + name: frequent-backup + repositories: + - backend: gcs-backend + directory: demo/sample-postgres-2 + encryptionSecret: + name: encrypt-secret + namespace: demo + name: customize-blueprint + scheduler: + jobTemplate: + backoffLimit: 1 + template: + controller: {} + metadata: {} + spec: + resources: {} + schedule: '*/10 * * * *' + sessionHistoryLimit: 3 + target: + apiGroup: kubedb.com + kind: Postgres + name: sample-postgres-2 + namespace: demo +status: + backends: + - name: gcs-backend + ready: true + retentionPolicy: + found: true + ref: + name: demo-retention + namespace: demo + storage: + phase: Ready + ref: + name: gcs-storage + namespace: demo + conditions: + - lastTransitionTime: "2024-09-05T12:39:37Z" + message: Validation has been passed successfully. + reason: ResourceValidationPassed + status: "True" + type: ValidationPassed + dependencies: + - found: true + kind: Addon + name: postgres-addon + phase: Ready + repositories: + - name: customize-blueprint + phase: Ready + sessions: + - conditions: + - lastTransitionTime: "2024-09-05T12:39:37Z" + message: Scheduler has been ensured successfully. + reason: SchedulerEnsured + status: "True" + type: SchedulerEnsured + - lastTransitionTime: "2024-09-05T12:39:37Z" + message: Initial backup has been triggered successfully. + reason: SuccessfullyTriggeredInitialBackup + status: "True" + type: InitialBackupTriggered + name: frequent-backup + targetFound: true +``` + +Notice the `spec.backends`, `spec.sessions` and `spec.target` sections, KubeStash automatically resolved those info from the `BackupBluePrint` and created above `BackupConfiguration`. + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +appbinding-sample-postgres-frequent-backup-1725597000 BackupConfiguration appbinding-sample-postgres Succeeded 58s 112s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `customize-blueprint` has been updated by the following command, + +```bash +$ kubectl get repository -n demo customize-blueprint +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +customize-blueprint true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=customize-blueprint +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +customize-blueprint-appbinding-ses-2-frequent-backup-1725597000 customize-blueprint frequent-backup 2024-09-06T04:30:00Z Delete Succeeded 6m19s +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubedb.com/db-version: ` +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo customize-blueprint-appbinding-sql-2-frequent-backup-1725597000 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-09-06T04:30:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubedb.com/db-version: "16.1" + kubestash.com/app-ref-kind: Postgres + kubestash.com/app-ref-name: sample-postgres-2 + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: customize-blueprint + name: customize-blueprint-appbinding-ses-2-frequent-backup-1725597000 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: customize-blueprint + uid: 5d4618c5-c28a-456a-9854-f6447161d3d1 + resourceVersion: "315624" + uid: 7e02a18c-c8a7-40be-bd22-e7312678d6f7 +spec: + appRef: + apiGroup: kubedb.com + kind: Postgres + name: sample-postgres-2 + namespace: demo + backupSession: appbinding-sample-postgres-2-frequent-backup-1725597000 + deletionPolicy: Delete + repository: customize-blueprint + session: frequent-backup + snapshotID: 01J72SH8XPEHB6SYNXFE00V5PB + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 7.060169632s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: 74d82943e0d676321e989edb503f5e2d6fe5cf4f4be72d386e492ec533358c26 + size: 1.220 KiB + uploaded: 296 B + size: 1.873 KiB + conditions: + - lastTransitionTime: "2024-09-06T04:30:00Z" + message: Recent snapshot list updated successfully + reason: SuccessfullyUpdatedRecentSnapshotList + status: "True" + type: RecentSnapshotListUpdated + - lastTransitionTime: "2024-09-06T04:30:38Z" + message: Metadata uploaded to backend successfully + reason: SuccessfullyUploadedSnapshotMetadata + status: "True" + type: SnapshotMetadataUploaded + integrity: true + phase: Succeeded + size: 1.872 KiB + snapshotTime: "2024-09-06T04:30:00Z" + totalComponents: 1 +``` + + +> KubeStash uses the `pg_dump` command to take backups of target PostgreSQL databases. Therefore, the component name for `logical backups` is set as `dump`. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `blueprint/demo/sample-postgres-2/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/demo/sample-postgres-2/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Cleanup + +To cleanup the resources crated by this tutorial, run the following commands, + +```bash +kubectl delete backupblueprints.core.kubestash.com -n demo postgres-default-backup-blueprint +kubectl delete backupblueprints.core.kubestash.com -n demo postgres-customize-backup-blueprint +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention +kubectl delete backupstorage -n demo gcs-storage +kubectl delete secret -n demo gcs-secret +kubectl delete secret -n demo encrypt-secret +kubectl delete postgres -n demo sample-postgres +kubectl delete postgres -n demo sample-postgres-2 +``` \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/backup/multiple-repository.yaml b/docs/guides/postgres/backup/kubestash/customization/backup/multiple-repository.yaml new file mode 100644 index 0000000000..25cffb0b05 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/backup/multiple-repository.yaml @@ -0,0 +1,42 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + - name: gcs-postgres-repo-2 + backend: gcs-backend + directory: /postgres-copy + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/backup/passing-args.yaml b/docs/guides/postgres/backup/kubestash/customization/backup/passing-args.yaml new file mode 100644 index 0000000000..5803c51a7e --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/backup/passing-args.yaml @@ -0,0 +1,38 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup + params: + args: --clean \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/backup/passing-database.yaml b/docs/guides/postgres/backup/kubestash/customization/backup/passing-database.yaml new file mode 100644 index 0000000000..6a6ceaf6cf --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/backup/passing-database.yaml @@ -0,0 +1,39 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup + params: + backupCmd: pg_dump + args: testdb \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/backup/resources-limit.yaml b/docs/guides/postgres/backup/kubestash/customization/backup/resources-limit.yaml new file mode 100644 index 0000000000..baadf69c50 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/backup/resources-limit.yaml @@ -0,0 +1,45 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/backup/specific-user.yaml b/docs/guides/postgres/backup/kubestash/customization/backup/specific-user.yaml new file mode 100644 index 0000000000..712b2cb4b0 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/backup/specific-user.yaml @@ -0,0 +1,41 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/common/backupstorage.yaml b/docs/guides/postgres/backup/kubestash/customization/common/backupstorage.yaml new file mode 100644 index 0000000000..0461b26762 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/common/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/common/retentionpolicy.yaml b/docs/guides/postgres/backup/kubestash/customization/common/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/common/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/common/sample-postgres.yaml b/docs/guides/postgres/backup/kubestash/customization/common/sample-postgres.yaml new file mode 100644 index 0000000000..4a1abc9f9a --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/common/sample-postgres.yaml @@ -0,0 +1,18 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: sample-postgres + namespace: demo +spec: + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/index.md b/docs/guides/postgres/backup/kubestash/customization/index.md new file mode 100644 index 0000000000..ef91b9ded2 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/index.md @@ -0,0 +1,387 @@ +--- +title: PostgreSQL Backup Customization | KubeStash +description: Customizing PostgreSQL Backup and Restore process with KubeStash +menu: + docs_{{ .version }}: + identifier: guides-pg-backup-customization-stashv2 + name: Customizing Backup & Restore Process + parent: guides-pg-backup-stashv2 + weight: 50 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Customizing Backup and Restore Process + +KubeStash provides rich customization supports for the backup and restore process to meet the requirements of various cluster configurations. This guide will show you some examples of these customizations. + +## Customizing Backup Process + +In this section, we are going to show you how to customize the backup process. Here, we are going to show some examples of providing arguments to the backup process, running the backup process as a specific user, etc. + +### Passing arguments to the backup process + +KubeStash PostgreSQL addon uses the [pg_dumpall](https://www.postgresql.org/docs/current/app-pg-dumpall.html) command by default for backups. However, you can change the dump command to [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html) by setting the `backupCmd` parameter under the `addon.tasks[*].params` section. You can pass supported options for either `pg_dumpall` or `pg_dump` through the `args` parameter in the same section. + +The below example shows how you can pass the `--clean` to include SQL commands to clean (drop) databases before recreating them. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup + params: + args: --clean +``` + + +### Passing a target database to the backup process + +KubeStash PostgreSQL addon uses the [pg_dumpall](https://www.postgresql.org/docs/current/app-pg-dumpall.html) command by default for backups. For a single database backup, you need to rewrite the dump command. You can do this by setting `backupCmd` to [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html) under the `addon.tasks[*].params` section and specifying the database name using the `args` parameter in the same section. + +The below example shows how you can set `pg_dump` and pass target database name during backup. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup + params: + backupCmd: pg_dump + args: testdb +``` + +> **WARNING**: Make sure that your provided database has been created before taking backup. + + +### Using multiple repositories + +You can configure multiple repositories for the same backend. For example, if you want to back up the `/postgres` directory using the `gcs-postgres-repo` repository, you can also back up another directory, such as `/postgres-copy`, by using a different repository, like `gcs-postgres-repo-2`. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + - name: gcs-postgres-repo-2 + backend: gcs-backend + directory: /postgres-copy + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup +``` + +### Running backup job as a specific user + +If your cluster requires running the backup job as a specific user, you can provide `securityContext` under `addon.jobTemplate.spec.securityContext` section. The below example shows how you can run the backup job as the `root` user. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup +``` + +### Specifying Memory/CPU limit/request for the backup job + +If you want to specify the Memory/CPU limit/request for your backup job, you can specify `resources` field under `addon.jobTemplate.spec` section. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup +``` + +> You can configure additional runtime settings for backup jobs within the `addon.jobTemplate.spec` sections. For further details, please refer to the [reference](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/#podtemplate-spec). + +## Customizing Restore Process + +`KubeStash` uses [psql](https://www.postgresql.org/docs/current/app-psql.html) during the restore process. In this section, we are going to show how you can pass arguments to the restore process, restore a specific snapshot, run restore job as a specific user, etc. + + +### Restore specific snapshot + +You can also restore a specific snapshot. At first, list the available snapshot as bellow, + +```bash +$ kubectl get snapshots.storage.kubestash.com -n demo -l=kubestash.com/repo-name=gcs-postgres-repo +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725257849 gcs-postgres-repo frequent-backup 2024-09-02T06:18:01Z Delete Succeeded 15m +gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725258000 gcs-postgres-repo frequent-backup 2024-09-02T06:20:00Z Delete Succeeded 13m +gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725258300 gcs-postgres-repo frequent-backup 2024-09-02T06:25:00Z Delete Succeeded 8m34s +gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725258600 gcs-postgres-repo frequent-backup 2024-09-02T06:30:00Z Delete Succeeded 3m34s +``` + +The below example shows how you can pass a specific snapshot name in `.spec.dataSource` section. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725258000 + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup-restore +``` + + +### Running restore job as a specific user + +Similar to the backup process under the `addon.jobTemplate.spec.` you can provide `securityContext` to run the restore job as a specific user. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup-restore +``` + +### Specifying Memory/CPU limit/request for the restore job + +Similar to the backup process, you can also provide `resources` field under the `addon.jobTemplate.spec.resources` section to limit the Memory/CPU for your restore job. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup-restore +``` + +> You can configure additional runtime settings for restore jobs within the `addon.jobTemplate.spec` sections. For further details, please refer to the [reference](https://kubestash.com/docs/latest/concepts/crds/restoresession/#podtemplate-spec). \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/restore/resources-limit.yaml b/docs/guides/postgres/backup/kubestash/customization/restore/resources-limit.yaml new file mode 100644 index 0000000000..f14e0c295b --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/restore/resources-limit.yaml @@ -0,0 +1,30 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup-restore \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/restore/specific-snapshot.yaml b/docs/guides/postgres/backup/kubestash/customization/restore/specific-snapshot.yaml new file mode 100644 index 0000000000..f77a99b0fd --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/restore/specific-snapshot.yaml @@ -0,0 +1,21 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725258000 + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup-restore \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/restore/specific-user.yaml b/docs/guides/postgres/backup/kubestash/customization/restore/specific-user.yaml new file mode 100644 index 0000000000..c712466b5f --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/restore/specific-user.yaml @@ -0,0 +1,26 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup-restore \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/logical/examples/backupconfiguration.yaml b/docs/guides/postgres/backup/kubestash/logical/examples/backupconfiguration.yaml new file mode 100644 index 0000000000..ddc74e3a7e --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/logical/examples/backupconfiguration.yaml @@ -0,0 +1,36 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml b/docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml new file mode 100644 index 0000000000..0461b26762 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/logical/examples/restored-postgres.yaml b/docs/guides/postgres/backup/kubestash/logical/examples/restored-postgres.yaml new file mode 100644 index 0000000000..7837574e49 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/logical/examples/restored-postgres.yaml @@ -0,0 +1,20 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: restored-postgres + namespace: demo +spec: + init: + waitForInitialRestore: true + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/logical/examples/restoresession.yaml b/docs/guides/postgres/backup/kubestash/logical/examples/restoresession.yaml new file mode 100644 index 0000000000..1bcb3e5a8c --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/logical/examples/restoresession.yaml @@ -0,0 +1,21 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup-restore \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/logical/examples/retentionpolicy.yaml b/docs/guides/postgres/backup/kubestash/logical/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/logical/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/logical/examples/sample-postgres.yaml b/docs/guides/postgres/backup/kubestash/logical/examples/sample-postgres.yaml new file mode 100644 index 0000000000..4a1abc9f9a --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/logical/examples/sample-postgres.yaml @@ -0,0 +1,18 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: sample-postgres + namespace: demo +spec: + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/logical/index.md b/docs/guides/postgres/backup/kubestash/logical/index.md new file mode 100644 index 0000000000..a0d0060b60 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/logical/index.md @@ -0,0 +1,760 @@ +--- +title: Backup & Restore PostgreSQL | KubeStash +description: Backup ans Restore PostgreSQL database using KubeStash +menu: + docs_{{ .version }}: + identifier: guides-pg-logical-backup-stashv2 + name: Logical Backup + parent: guides-pg-backup-stashv2 + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Backup and Restore PostgreSQL database using KubeStash + +KubeStash allows you to backup and restore `PostgreSQL` databases. It supports backups for `PostgreSQL` instances running in Standalone, and HA cluster configurations. KubeStash makes managing your `PostgreSQL` backups and restorations more straightforward and efficient. + +This guide will give you an overview how you can take backup and restore your `PostgreSQL` databases using `Kubestash`. + + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. +- Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). +- Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). +- Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- If you are not familiar with how KubeStash backup and restore PostgreSQL databases, please check the following guide [here](/docs/guides/postgres/backup/kubestash/overview/index.md). + +You should be familiar with the following `KubeStash` concepts: + +- [BackupStorage](https://kubestash.com/docs/latest/concepts/crds/backupstorage/) +- [BackupConfiguration](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/) +- [BackupSession](https://kubestash.com/docs/latest/concepts/crds/backupsession/) +- [RestoreSession](https://kubestash.com/docs/latest/concepts/crds/restoresession/) +- [Addon](https://kubestash.com/docs/latest/concepts/crds/addon/) +- [Function](https://kubestash.com/docs/latest/concepts/crds/function/) +- [Task](https://kubestash.com/docs/latest/concepts/crds/addon/#task-specification) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/guides/postgres/backup/kubestash/logical/examples](https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + + +## Backup PostgreSQL + +KubeStash supports backups for `PostgreSQL` instances across different configurations, including Standalone and HA Cluster setups. In this demonstration, we'll focus on a `PostgreSQL` database using HA cluster configuration. The backup and restore process is similar for Standalone configuration. + +This section will demonstrate how to backup a `PostgreSQL` database. Here, we are going to deploy a `PostgreSQL` database using KubeDB. Then, we are going to backup this database into a `GCS` bucket. Finally, we are going to restore the backup up data into another `PostgreSQL` database. + + +### Deploy Sample PostgreSQL Database + +Let's deploy a sample `PostgreSQL` database and insert some data into it. + +**Create PostgreSQL CR:** + +Below is the YAML of a sample `PostgreSQL` CR that we are going to create for this tutorial: + +```yaml +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: sample-postgres + namespace: demo +spec: + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +Create the above `PostgreSQL` CR, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples/sample-postgres.yaml +postgres.kubedb.com/sample-postgres created +``` + +KubeDB will deploy a `PostgreSQL` database according to the above specification. It will also create the necessary `Secrets` and `Services` to access the database. + +Let's check if the database is ready to use, + +```bash +$ kubectl get pg -n demo sample-postgres +NAME VERSION STATUS AGE +sample-postgres 16.1 Ready 5m1s +``` + +The database is `Ready`. Verify that KubeDB has created a `Secret` and a `Service` for this database using the following commands, + +```bash +$ kubectl get secret -n demo +NAME TYPE DATA AGE +sample-postgres-auth kubernetes.io/basic-auth 2 5m20s + +$ kubectl get service -n demo -l=app.kubernetes.io/instance=sample-postgres +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +sample-postgres ClusterIP 10.96.23.177 5432/TCP,2379/TCP 5m55s +sample-postgres-pods ClusterIP None 5432/TCP,2380/TCP,2379/TCP 5m55s +sample-postgres-standby ClusterIP 10.96.26.118 5432/TCP 5m55s +``` + +Here, we have to use service `sample-postgres` and secret `sample-postgres-auth` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/postgres/concepts/appbinding.md) CR that holds the necessary information to connect with the database. + + +**Verify AppBinding:** + +Verify that the `AppBinding` has been created successfully using the following command, + +```bash +$ kubectl get appbindings -n demo +NAME TYPE VERSION AGE +sample-postgres kubedb.com/postgres 16.1 9m30s +``` + +Let's check the YAML of the above `AppBinding`, + +```bash +$ kubectl get appbindings -n demo sample-postgres -o yaml +``` + +```yaml +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"kubedb.com/v1","kind":"Postgres","metadata":{"annotations":{},"name":"sample-postgres","namespace":"demo"},"spec":{"deletionPolicy":"DoNotTerminate","replicas":3,"standbyMode":"Hot","storage":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"1Gi"}}},"storageType":"Durable","streamingMode":"Synchronous","version":"16.1"}} + creationTimestamp: "2024-09-04T10:07:04Z" + generation: 1 + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: sample-postgres + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: postgreses.kubedb.com + name: sample-postgres + namespace: demo + ownerReferences: + - apiVersion: kubedb.com/v1 + blockOwnerDeletion: true + controller: true + kind: Postgres + name: sample-postgres + uid: 0810a96c-a2b6-4e8a-a70a-51753660450c + resourceVersion: "245972" + uid: 73bdba85-c932-464b-93a8-7f1ba8dfff1b +spec: + appRef: + apiGroup: kubedb.com + kind: Postgres + name: sample-postgres + namespace: demo + clientConfig: + service: + name: sample-postgres + path: / + port: 5432 + query: sslmode=disable + scheme: postgresql + parameters: + apiVersion: appcatalog.appscode.com/v1alpha1 + kind: StashAddon + stash: + addon: + backupTask: + name: postgres-backup-16.1 + restoreTask: + name: postgres-restore-16.1 + secret: + name: sample-postgres-auth + type: kubedb.com/postgres + version: "16.1" +``` + +KubeStash uses the `AppBinding` CR to connect with the target database. It requires the following two fields to set in AppBinding's `.spec` section. + +Here, + +- `.spec.clientConfig.service.name` specifies the name of the Service that connects to the database. +- `.spec.secret` specifies the name of the Secret that holds necessary credentials to access the database. +- `.spec.type` specifies the types of the app that this AppBinding is pointing to. KubeDB generated AppBinding follows the following format: `/`. + + +**Insert Sample Data:** + +Now, we are going to exec into one of the database pod and create some sample data. At first, find out the database `Pod` using the following command, + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-postgres" +NAME READY STATUS RESTARTS AGE +sample-postgres-0 2/2 Running 0 16m +sample-postgres-1 2/2 Running 0 13m +sample-postgres-2 2/2 Running 0 13m +``` + +Now, let’s exec into the pod and create a table, + +```bash +$ kubectl exec -it -n demo sample-postgres-0 -- sh + +# login as "postgres" superuser. +/ $ psql -U postgres +psql (16.1) +Type "help" for help. + +# list available databases +postgres=# \l + List of databases + Name | Owner | Encoding | Locale Provider | Collate | Ctype | ICU Locale | ICU Rules | Access privileges +---------------+----------+----------+-----------------+------------+------------+------------+-----------+----------------------- + kubedb_system | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + postgres | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + template0 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres + template1 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres +(4 rows) + +# create a database named "demo" +postgres=# create database demo; +CREATE DATABASE + +# verify that the "demo" database has been created +postgres=# \l + List of databases + Name | Owner | Encoding | Locale Provider | Collate | Ctype | ICU Locale | ICU Rules | Access privileges +---------------+----------+----------+-----------------+------------+------------+------------+-----------+----------------------- + demo | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + kubedb_system | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + postgres | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + template0 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres + template1 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres +(5 rows) + +# connect to the "demo" database +postgres=# \c demo +You are now connected to database "demo" as user "postgres". + +# create a sample table +demo=# CREATE TABLE COMPANY( NAME TEXT NOT NULL, EMPLOYEE INT NOT NULL); +CREATE TABLE + +# verify that the table has been created +demo=# \d + List of relations + Schema | Name | Type | Owner +--------+---------+-------+---------- + public | company | table | postgres +(1 row) + +# quit from the database +demo=# \q + +# exit from the pod +/ $ exit +``` + +Now, we are ready to backup the database. + +### Prepare Backend + +We are going to store our backed up data into a `GCS` bucket. We have to create a `Secret` with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). + +**Create Secret:** + +Let's create a secret called `gcs-secret` with access credentials to our desired GCS bucket, + +```bash +$ echo -n '' > GOOGLE_PROJECT_ID +$ cat /path/to/downloaded-sa-key.json > GOOGLE_SERVICE_ACCOUNT_JSON_KEY +$ kubectl create secret generic -n demo gcs-secret \ + --from-file=./GOOGLE_PROJECT_ID \ + --from-file=./GOOGLE_SERVICE_ACCOUNT_JSON_KEY +secret/gcs-secret created +``` + +**Create BackupStorage:** + +Now, create a `BackupStorage` using this secret. Below is the YAML of `BackupStorage` CR we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: WipeOut +``` + +Let's create the BackupStorage we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml +backupstorage.storage.kubestash.com/gcs-storage created +``` + +Now, we are ready to backup our database to our desired backend. + +**Create RetentionPolicy:** + +Now, let's create a `RetentionPolicy` to specify how the old Snapshots should be cleaned up. + +Below is the YAML of the `RetentionPolicy` object that we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Let’s create the above `RetentionPolicy`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples/retentionpolicy.yaml +retentionpolicy.storage.kubestash.com/demo-retention created +``` + +### Backup + +We have to create a `BackupConfiguration` targeting respective `sample-postgres` PostgreSQL database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. + +At first, we need to create a secret with a Restic password for backup data encryption. + +**Create Secret:** + +Let's create a secret called `encrypt-secret` with the Restic password, + +```bash +$ echo -n 'changeit' > RESTIC_PASSWORD +$ kubectl create secret generic -n demo encrypt-secret \ + --from-file=./RESTIC_PASSWORD \ +secret "encrypt-secret" created +``` + +Below is the YAML for `BackupConfiguration` CR to backup the `sample-postgres` database that we have deployed earlier, + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-postgres-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: sample-postgres + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-postgres-repo + backend: gcs-backend + directory: /postgres + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup +``` + +- `.spec.sessions[*].schedule` specifies that we want to backup the database at `5 minutes` interval. +- `.spec.target` refers to the targeted `sample-postgres` PostgreSQL database that we created earlier. + +Let's create the `BackupConfiguration` CR that we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/kubestash/logical/examples/backupconfiguration.yaml +backupconfiguration.core.kubestash.com/sample-postgres-backup created +``` + +**Verify Backup Setup Successful** + +If everything goes well, the phase of the `BackupConfiguration` should be `Ready`. The `Ready` phase indicates that the backup setup is successful. Let's verify the `Phase` of the BackupConfiguration, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +sample-postgres-backup Ready 2m50s +``` + +Additionally, we can verify that the `Repository` specified in the `BackupConfiguration` has been created using the following command, + +```bash +$ kubectl get repo -n demo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-postgres-repo 0 0 B Ready 3m +``` + +KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/postgres` directory. + +**Verify CronJob:** + +It will also create a `CronJob` with the schedule specified in `spec.sessions[*].scheduler.schedule` field of `BackupConfiguration` CR. + +Verify that the `CronJob` has been created using the following command, + +```bash +$ kubectl get cronjob -n demo +NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE +trigger-sample-postgres-backup-frequent-backup */5 * * * * 0 2m45s 3m25s +``` + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +sample-postgres-backup-frequent-backup-1725449400 BackupConfiguration sample-postgres-backup Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-postgres-backup` has been updated by the following command, + +```bash +$ kubectl get repository -n demo sample-postgres-backup +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +sample-postgres-backup true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=gcs-postgres-repo +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725449400 gcs-postgres-repo frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubedb.com/db-version: ` +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725449400 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-09-04T11:30:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubedb.com/db-version: "16.1" + kubestash.com/app-ref-kind: Postgres + kubestash.com/app-ref-name: sample-postgres + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: gcs-postgres-repo + name: gcs-postgres-repo-sample-postgreckup-frequent-backup-1725449400 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: gcs-postgres-repo + uid: 1009bd4a-b211-49f1-a64c-3c979c699a81 + resourceVersion: "253523" + uid: c6757c49-e13b-4a36-9f7d-64eae350423f +spec: + appRef: + apiGroup: kubedb.com + kind: Postgres + name: sample-postgres + namespace: demo + backupSession: sample-postgres-backup-frequent-backup-1725449400 + deletionPolicy: Delete + repository: gcs-postgres-repo + session: frequent-backup + snapshotID: 01J6YCRWEWAKACMGZYR2R7YJ5C + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 11.526138009s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: 008eb87193e7db112e9ad8f42c9302c851a1fbacb7165a5cb3aa2d27dd210764 + size: 3.345 KiB + uploaded: 299 B + size: 2.202 KiB + conditions: + - lastTransitionTime: "2024-09-04T11:30:00Z" + message: Recent snapshot list updated successfully + reason: SuccessfullyUpdatedRecentSnapshotList + status: "True" + type: RecentSnapshotListUpdated + - lastTransitionTime: "2024-09-04T11:30:32Z" + message: Metadata uploaded to backend successfully + reason: SuccessfullyUploadedSnapshotMetadata + status: "True" + type: SnapshotMetadataUploaded + integrity: true + phase: Succeeded + size: 2.201 KiB + snapshotTime: "2024-09-04T11:30:00Z" + totalComponents: 1 +``` + +> KubeStash uses the `pg_dump` command to take backups of target PostgreSQL databases. Therefore, the component name for `logical backups` is set as `dump`. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/popstgres/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/postgres/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + + +## Restore + +In this section, we are going to restore the database from the backup we have taken in the previous section. We are going to deploy a new database and initialize it from the backup. + +Now, we have to deploy the restored database similarly as we have deployed the original `sample-postgres` database. However, this time there will be the following differences: + +- We are going to specify `.spec.init.waitForInitialRestore` field that tells KubeDB to wait for first restore to complete before marking this database is ready to use. + +Below is the YAML for `PostgreSQL` CR we are going deploy to initialize from backup, + +```yaml +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: restored-postgres + namespace: demo +spec: + init: + waitForInitialRestore: true + version: "16.1" + replicas: 3 + standbyMode: Hot + streamingMode: Synchronous + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +Let's create the above database, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples/restored-postgres.yaml +postgres.kubedb.com/restore-postgres created +``` + +If you check the database status, you will see it is stuck in **`Provisioning`** state. + +```bash +$ kubectl get postgres -n demo restored-postgres +NAME VERSION STATUS AGE +restored-postgres 8.2.0 Provisioning 61s +``` + +#### Create RestoreSession: + +Now, we need to create a `RestoreSession` CR pointing to targeted `PostgreSQL` database. + +Below, is the contents of YAML file of the `RestoreSession` object that we are going to create to restore backed up data into the newly created `PostgreSQL` database named `restored-postgres`. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup-restore +``` + +Here, + +- `.spec.target` refers to the newly created `restored-postgres` PostgreSQL object to where we want to restore backup data. +- `.spec.dataSource.repository` specifies the Repository object that holds the backed up data. +- `.spec.dataSource.snapshot` specifies to restore from latest `Snapshot`. + +Let's create the RestoreSession CRD object we have shown above, + +```bash +$ kubectl apply -f **https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples/restoresession.yaml +restoresession.core.kubestash.com/sample-postgres-restore created +``` + +Once, you have created the `RestoreSession` object, KubeStash will create restore Job. Run the following command to watch the phase of the `RestoreSession` object, + +```bash +$ watch kubectl get restoresession -n demo +Every 2.0s: kubectl get restores... AppsCode-PC-03: Wed Aug 21 10:44:05 2024 +NAME REPOSITORY FAILURE-POLICY PHASE DURATION AGE +sample-postgres-restore gcs-postgres-repo Succeeded 7s 116s +``` + +The `Succeeded` phase means that the restore process has been completed successfully. + +#### Verify Restored Data: + +In this section, we are going to verify whether the desired data has been restored successfully. We are going to connect to the database server and check whether the database and the table we created earlier in the original database are restored. + +At first, check if the database has gone into **`Ready`** state by the following command, + +```bash +$ kubectl get postgres -n demo restored-postgres +NAME VERSION STATUS AGE +restored-postgres 16.1 Ready 6m31s +``` + +Now, find out the database `Pod` by the following command, + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=restored-postgres" +NAME READY STATUS RESTARTS AGE +restored-postgres-0 2/2 Running 0 6m7s +restored-postgres-1 2/2 Running 0 6m1s +restored-postgres-2 2/2 Running 0 5m55s +``` + +Now, lets exec one of the `Pod` and verify restored data. + +```bash +$ kubectl exec -it -n demo restored-postgres-0 -- /bin/sh +# login as "postgres" superuser. +/ # psql -U postgres +psql (11.11) +Type "help" for help. + +# verify that the "demo" database has been restored +postgres=# \l + List of databases + Name | Owner | Encoding | Locale Provider | Collate | Ctype | ICU Locale | ICU Rules | Access privileges +---------------+----------+----------+-----------------+------------+------------+------------+-----------+----------------------- + demo | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + kubedb_system | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + postgres | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | + template0 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres + template1 | postgres | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/postgres + + | | | | | | | | postgres=CTc/postgres +(5 rows) + +# connect to the "demo" database +postgres=# \c demo +You are now connected to database "demo" as user "postgres". + +# verify that the sample table has been restored +demo=# \d + List of relations + Schema | Name | Type | Owner +--------+---------+-------+---------- + public | company | table | postgres +(1 row) + +# disconnect from the database +demo=# \q + +# exit from the pod +/ # exit +``` + +So, from the above output, we can see the `demo` database we had created in the original database `sample-postgres` has been restored in the `restored-postgres` database. + + +## Cleanup + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl delete backupconfigurations.core.kubestash.com -n demo sample-postgres-backup +kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-postgres +kubectl delete backupstorage -n demo gcs-storage +kubectl delete secret -n demo gcs-secret +kubectl delete secret -n demo encrypt-secret +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention +kubectl delete postgres -n demo restored-postgres +kubectl delete postgres -n demo sample-postgres +``` \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/overview/images/backup_overview.svg b/docs/guides/postgres/backup/kubestash/overview/images/backup_overview.svg new file mode 100644 index 0000000000..9d245e0dcd --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/overview/images/backup_overview.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/overview/images/kubedb_plus_kubestash.svg b/docs/guides/postgres/backup/kubestash/overview/images/kubedb_plus_kubestash.svg new file mode 100644 index 0000000000..380d92d969 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/overview/images/kubedb_plus_kubestash.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/guides/postgres/backup/kubestash/overview/images/restore_overview.svg b/docs/guides/postgres/backup/kubestash/overview/images/restore_overview.svg new file mode 100644 index 0000000000..f2c2488962 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/overview/images/restore_overview.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/overview/index.md b/docs/guides/postgres/backup/kubestash/overview/index.md new file mode 100644 index 0000000000..8d82725ef2 --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/overview/index.md @@ -0,0 +1,98 @@ +--- +title: Backup & Restore PostgreSQL Using KubeStash +menu: + docs_{{ .version }}: + identifier: guides-pg-backup-overview-stashv2 + name: Overview + parent: guides-pg-backup-stashv2 + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +{{< notice type="warning" message="Please install [KubeStash](https://kubestash.com/docs/latest/setup/install/kubestash/) to try this feature. Database backup with KubeStash is already included in the KubeDB license. So, you don't need a separate license for KubeStash." >}} + +# PostgreSQL Backup & Restore Overview + +KubeDB also uses [KubeStash](https://kubestash.com) to backup and restore databases. KubeStash by AppsCode is a cloud native data backup and recovery solution for Kubernetes workloads and databases. KubeStash utilizes [restic](https://github.com/restic/restic) to securely backup stateful applications to any cloud or on-prem storage backends (for example, S3, GCS, Azure Blob storage, Minio, NetApp, Dell EMC etc.). + +
+  KubeDB + KubeStash +
Fig: Backup KubeDB Databases Using KubeStash
+
+ +## How Backup Works + +The following diagram shows how KubeStash takes backup of a `PostgreSQL` database. Open the image in a new tab to see the enlarged version. + +
+  PostgreSQL Backup Overview +
Fig: PostgreSQL Backup Overview
+
+ +The backup process consists of the following steps: + +1. At first, a user creates a `Secret`. This secret holds the credentials to access the backend where the backed up data will be stored. + +2. Then, she creates a `BackupStorage` custom resource that specifies the backend information, along with the `Secret` containing the credentials needed to access the backend. + +3. KubeStash operator watches for `BackupStorage` custom resources. When it finds a `BackupStorage` object, it initializes the `BackupStorage` by uploading the `metadata.yaml` file into the target storage. + +4. Then, she creates a `BackupConfiguration` custom resource that specifies the targeted the KubeDB managed `PostgreSQL` database, the `Addon` info with a specified task, etc. It also provides information about one or more repositories, each indicating a path and a `BackupStorage` for storing the backed-up data. + +5. KubeStash operator watches for `BackupConfiguration` objects. + +6. Once the KubeStash operator finds a `BackupConfiguration` object, it creates `Repository` with the information specified in the `BackupConfiguration`. + +7. KubeStash operator watches for `Repository` custom resources. When it finds the `Repository` object, it Initializes `Repository` by uploading `repository.yaml` file into the `spec.sessions[*].repositories[*].directory` path specified in `BackupConfiguration`. + +8. Then, it creates a `CronJob` for each session with the schedule specified in `BackupConfiguration` to trigger backup periodically. + +9. KubeStash operator triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are triggered by the `CronJob` according to the specified schedule. + +10. KubeStash operator watches for `BackupSession` custom resources. + +11. When it finds a `BackupSession` object, it creates a `Snapshot` custom resource for each `Repository` specified in the `BackupConfiguration`. + +12. Then it resolves the respective `Addon` and `Function` and prepares backup `Job` definition. + +13. Then, it creates the `Job` to backup the targeted `PostgreSQL` database. + +14. The backup `Job` reads necessary information (e.g. auth secret, port) to connect with the database from the `AppBinding` crd. It also reads backend information and access credentials from BackupStorage crd, Storage Secret and Repository path respectively. + +15. Then, the `Job` dumps the targeted `PostgreSQL` database and uploads the output to the backend. KubeStash pipes the output of dump command to uploading process. Hence, backup `Job` does not require a large volume to hold the entire dump output. + +16. After the backup process is completed, the backup `Job` updates the `status.components[dump]` field of the `Snapshot` resources with backup information of the target `PostgreSQL` database. + +## How Restore Process Works + +The following diagram shows how KubeStash restores backed up data into a `PostgreSQL` database. Open the image in a new tab to see the enlarged version. + +
+  Database Restore Overview +
Fig: PostgreSQL Restore Process Overview
+
+ +The restore process consists of the following steps: + +1. At first, a user creates a `PostgreSQL` database where the data will be restored or the user can use the same `PostgreSQL` database. + +2. Then, she creates a `RestoreSession` custom resource that specifies the target `PostgreSQL` database where the backed-up data will be restored. the `Repository` object that points to a `BackupStorage` that holds backend information, and the target `Snapshot`, which will be restored. It also specifies the `Addon` info with task to use to restore. + +3. KubeStash operator watches for `RestoreSession` custom resources. + +4. When it finds a `RestoreSession` custom resource, it resolves the respective `Addon` and `Function` and prepares a restore `Job` definition. + +5. Then, it creates the `Job` to restore the target. + +6. The `Job` reads necessary information to connect with the database from respective `AppBinding` crd. It also reads backend information and access credentials from `Repository` crd and storage `Secret` respectively. + +7. Then, the `Job` downloads the backed up data from the backend and injects into the desired database. KubeStash pipes the downloaded data to the respective database tool to inject into the database. Hence, restore `Job` does not require a large volume to download entire backup data inside it. + +8. Finally, when the restore process is completed, the `Job` updates the `status.components[dump]` field of the `RestoreSession` with restore information of the target database. + +## Next Steps + +- Backup a `PostgreSQL` database using KubeStash by the following guides from [here](/docs/guides/postgres/backup/kubestash/logical/index.md). \ No newline at end of file diff --git a/docs/guides/postgres/backup/stash/_index.md b/docs/guides/postgres/backup/stash/_index.md new file mode 100644 index 0000000000..7508f6b8ca --- /dev/null +++ b/docs/guides/postgres/backup/stash/_index.md @@ -0,0 +1,10 @@ +--- +title: Backup & Restore PostgreSQL | Stash +menu: + docs_{{ .version }}: + identifier: guides-pg-backup-stashv1 + name: Stash + parent: guides-pg-backup + weight: 40 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/postgres/backup/auto-backup/examples/backupblueprint.yaml b/docs/guides/postgres/backup/stash/auto-backup/examples/backupblueprint.yaml similarity index 100% rename from docs/guides/postgres/backup/auto-backup/examples/backupblueprint.yaml rename to docs/guides/postgres/backup/stash/auto-backup/examples/backupblueprint.yaml diff --git a/docs/guides/postgres/backup/auto-backup/examples/sample-pg-1.yaml b/docs/guides/postgres/backup/stash/auto-backup/examples/sample-pg-1.yaml similarity index 100% rename from docs/guides/postgres/backup/auto-backup/examples/sample-pg-1.yaml rename to docs/guides/postgres/backup/stash/auto-backup/examples/sample-pg-1.yaml diff --git a/docs/guides/postgres/backup/auto-backup/examples/sample-pg-2.yaml b/docs/guides/postgres/backup/stash/auto-backup/examples/sample-pg-2.yaml similarity index 100% rename from docs/guides/postgres/backup/auto-backup/examples/sample-pg-2.yaml rename to docs/guides/postgres/backup/stash/auto-backup/examples/sample-pg-2.yaml diff --git a/docs/guides/postgres/backup/auto-backup/examples/sample-pg-3.yaml b/docs/guides/postgres/backup/stash/auto-backup/examples/sample-pg-3.yaml similarity index 100% rename from docs/guides/postgres/backup/auto-backup/examples/sample-pg-3.yaml rename to docs/guides/postgres/backup/stash/auto-backup/examples/sample-pg-3.yaml diff --git a/docs/guides/postgres/backup/auto-backup/images/sample-postgres-1.png b/docs/guides/postgres/backup/stash/auto-backup/images/sample-postgres-1.png similarity index 100% rename from docs/guides/postgres/backup/auto-backup/images/sample-postgres-1.png rename to docs/guides/postgres/backup/stash/auto-backup/images/sample-postgres-1.png diff --git a/docs/guides/postgres/backup/auto-backup/images/sample-postgres-2.png b/docs/guides/postgres/backup/stash/auto-backup/images/sample-postgres-2.png similarity index 100% rename from docs/guides/postgres/backup/auto-backup/images/sample-postgres-2.png rename to docs/guides/postgres/backup/stash/auto-backup/images/sample-postgres-2.png diff --git a/docs/guides/postgres/backup/auto-backup/images/sample-postgres-3.png b/docs/guides/postgres/backup/stash/auto-backup/images/sample-postgres-3.png similarity index 100% rename from docs/guides/postgres/backup/auto-backup/images/sample-postgres-3.png rename to docs/guides/postgres/backup/stash/auto-backup/images/sample-postgres-3.png diff --git a/docs/guides/postgres/backup/auto-backup/index.md b/docs/guides/postgres/backup/stash/auto-backup/index.md similarity index 96% rename from docs/guides/postgres/backup/auto-backup/index.md rename to docs/guides/postgres/backup/stash/auto-backup/index.md index 08dbc582d1..b90494dbc4 100644 --- a/docs/guides/postgres/backup/auto-backup/index.md +++ b/docs/guides/postgres/backup/stash/auto-backup/index.md @@ -3,9 +3,9 @@ title: PostgreSQL | Stash description: Stash auto-backup for PostgreSQL database menu: docs_{{ .version }}: - identifier: guides-pg-backup-auto-backup + identifier: guides-pg-backup-auto-backup-stashv1 name: Auto-Backup - parent: guides-pg-backup + parent: guides-pg-backup-stashv1 weight: 30 menu_name: docs_{{ .version }} section_menu_id: guides @@ -22,7 +22,7 @@ In this tutorial, we are going to show how you can configure a backup blueprint - At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. - Install KubeDB in your cluster following the steps [here](/docs/setup/README.md). - Install Stash in your cluster following the steps [here](https://stash.run/docs/latest/setup/install/stash/). -- If you are not familiar with how Stash backup and restore PostgreSQL databases, please check the following guide [here](/docs/guides/postgres/backup/overview/index.md). +- If you are not familiar with how Stash backup and restore PostgreSQL databases, please check the following guide [here](/docs/guides/postgres/backup/stash/overview/index.md). - If you are not familiar with how auto-backup works in Stash, please check the following guide [here](https://stash.run/docs/latest/guides/auto-backup/overview/). - If you are not familiar with the available auto-backup options for databases in Stash, please check the following guide [here](https://stash.run/docs/latest/guides/auto-backup/database/). @@ -99,7 +99,7 @@ Notice the `prefix` field of `backend` section. We have used some variables in f Let's create the `BackupBlueprint` we have shown above, ```bash -❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/auto-backup/examples/backupblueprint.yaml +❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/auto-backup/examples/backupblueprint.yaml backupblueprint.stash.appscode.com/postgres-backup-template created ``` @@ -154,7 +154,7 @@ Notice the `annotations` section. We are pointing to the `BackupBlueprint` that Let's create the above Postgres CRO, ```bash -❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/auto-backup/examples/sample-pg-1.yaml +❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/auto-backup/examples/sample-pg-1.yaml postgres.kubedb.com/sample-postgres-1 created ``` @@ -272,7 +272,7 @@ app-sample-postgres-1-1614073215 BackupConfiguration app-sample-postgres-1 Once the backup has been completed successfully, you should see the backed-up data has been stored in the bucket at the directory pointed by the `prefix` field of the `Repository`.
- Backup data in GCS Bucket + Backup data in GCS Bucket
Fig: Backup data in GCS Bucket
@@ -323,7 +323,7 @@ Notice the `annotations` section. This time, we have passed a schedule via `stas Let's create the above Postgres CRO, ```bash -❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/auto-backup/examples/sample-pg-2.yaml +❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/auto-backup/examples/sample-pg-2.yaml postgres.kubedb.com/sample-postgres-2 created ``` @@ -442,7 +442,7 @@ app-sample-postgres-2-1614073502 BackupConfiguration app-sample-postgres-2 Once the backup has been completed successfully, you should see that Stash has created a new directory as pointed by the `prefix` field of the new `Repository` and stored the backed-up data there.
- Backup data in GCS Bucket + Backup data in GCS Bucket
Fig: Backup data in GCS Bucket
@@ -493,7 +493,7 @@ Notice the `annotations` section. This time, we have passed an argument via `par Let's create the above Postgres CRO, ```bash -❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/auto-backup/examples/sample-pg-3.yaml +❯ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/auto-backup/examples/sample-pg-3.yaml postgres.kubedb.com/sample-postgres-3 created ``` @@ -616,7 +616,7 @@ app-sample-postgres-3-1614073808 BackupConfiguration app-sample-postgres-3 Once the backup has been completed successfully, you should see that Stash has created a new directory as pointed by the `prefix` field of the new `Repository` and stored the backed-up data there.
- Backup data in GCS Bucket + Backup data in GCS Bucket
Fig: Backup data in GCS Bucket
@@ -625,7 +625,7 @@ Once the backup has been completed successfully, you should see that Stash has c To cleanup the resources crated by this tutorial, run the following commands, ```bash -❯ kubectl delete -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/auto-backup/examples/ +❯ kubectl delete -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/auto-backup/examples/ backupblueprint.stash.appscode.com "postgres-backup-template" deleted postgres.kubedb.com "sample-postgres-1" deleted postgres.kubedb.com "sample-postgres-2" deleted diff --git a/docs/guides/postgres/backup/customization/examples/backup/multi-retention-policy.yaml b/docs/guides/postgres/backup/stash/customization/examples/backup/multi-retention-policy.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/backup/multi-retention-policy.yaml rename to docs/guides/postgres/backup/stash/customization/examples/backup/multi-retention-policy.yaml diff --git a/docs/guides/postgres/backup/customization/examples/backup/resource-limit.yaml b/docs/guides/postgres/backup/stash/customization/examples/backup/resource-limit.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/backup/resource-limit.yaml rename to docs/guides/postgres/backup/stash/customization/examples/backup/resource-limit.yaml diff --git a/docs/guides/postgres/backup/customization/examples/backup/specific-database-user.yaml b/docs/guides/postgres/backup/stash/customization/examples/backup/specific-database-user.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/backup/specific-database-user.yaml rename to docs/guides/postgres/backup/stash/customization/examples/backup/specific-database-user.yaml diff --git a/docs/guides/postgres/backup/customization/examples/backup/specific-user.yaml b/docs/guides/postgres/backup/stash/customization/examples/backup/specific-user.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/backup/specific-user.yaml rename to docs/guides/postgres/backup/stash/customization/examples/backup/specific-user.yaml diff --git a/docs/guides/postgres/backup/customization/examples/repository.yaml b/docs/guides/postgres/backup/stash/customization/examples/repository.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/repository.yaml rename to docs/guides/postgres/backup/stash/customization/examples/repository.yaml diff --git a/docs/guides/postgres/backup/customization/examples/restore/passing-args.yaml b/docs/guides/postgres/backup/stash/customization/examples/restore/passing-args.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/restore/passing-args.yaml rename to docs/guides/postgres/backup/stash/customization/examples/restore/passing-args.yaml diff --git a/docs/guides/postgres/backup/customization/examples/restore/resource-limit.yaml b/docs/guides/postgres/backup/stash/customization/examples/restore/resource-limit.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/restore/resource-limit.yaml rename to docs/guides/postgres/backup/stash/customization/examples/restore/resource-limit.yaml diff --git a/docs/guides/postgres/backup/customization/examples/restore/specific-database-user.yaml b/docs/guides/postgres/backup/stash/customization/examples/restore/specific-database-user.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/restore/specific-database-user.yaml rename to docs/guides/postgres/backup/stash/customization/examples/restore/specific-database-user.yaml diff --git a/docs/guides/postgres/backup/customization/examples/restore/specific-snapshot.yaml b/docs/guides/postgres/backup/stash/customization/examples/restore/specific-snapshot.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/restore/specific-snapshot.yaml rename to docs/guides/postgres/backup/stash/customization/examples/restore/specific-snapshot.yaml diff --git a/docs/guides/postgres/backup/customization/examples/restore/specific-user.yaml b/docs/guides/postgres/backup/stash/customization/examples/restore/specific-user.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/restore/specific-user.yaml rename to docs/guides/postgres/backup/stash/customization/examples/restore/specific-user.yaml diff --git a/docs/guides/postgres/backup/customization/examples/sample-postgres.yaml b/docs/guides/postgres/backup/stash/customization/examples/sample-postgres.yaml similarity index 100% rename from docs/guides/postgres/backup/customization/examples/sample-postgres.yaml rename to docs/guides/postgres/backup/stash/customization/examples/sample-postgres.yaml diff --git a/docs/guides/postgres/backup/customization/index.md b/docs/guides/postgres/backup/stash/customization/index.md similarity index 99% rename from docs/guides/postgres/backup/customization/index.md rename to docs/guides/postgres/backup/stash/customization/index.md index f40afa848b..51758c5e99 100644 --- a/docs/guides/postgres/backup/customization/index.md +++ b/docs/guides/postgres/backup/stash/customization/index.md @@ -3,9 +3,9 @@ title: PostgreSQL Backup Customization | Stash description: Customizing PostgreSQL Backup and Restore process with Stash menu: docs_{{ .version }}: - identifier: guides-pg-backup-customization + identifier: guides-pg-backup-customization-stashv1 name: Customizing Backup & Restore Process - parent: guides-pg-backup + parent: guides-pg-backup-stashv1 weight: 40 menu_name: docs_{{ .version }} section_menu_id: guides diff --git a/docs/guides/postgres/backup/overview/images/backup_overview.svg b/docs/guides/postgres/backup/stash/overview/images/backup_overview.svg similarity index 100% rename from docs/guides/postgres/backup/overview/images/backup_overview.svg rename to docs/guides/postgres/backup/stash/overview/images/backup_overview.svg diff --git a/docs/guides/postgres/backup/overview/images/restore_overview.svg b/docs/guides/postgres/backup/stash/overview/images/restore_overview.svg similarity index 100% rename from docs/guides/postgres/backup/overview/images/restore_overview.svg rename to docs/guides/postgres/backup/stash/overview/images/restore_overview.svg diff --git a/docs/guides/postgres/backup/overview/index.md b/docs/guides/postgres/backup/stash/overview/index.md similarity index 93% rename from docs/guides/postgres/backup/overview/index.md rename to docs/guides/postgres/backup/stash/overview/index.md index c48a7c6cfa..cb56b2b426 100644 --- a/docs/guides/postgres/backup/overview/index.md +++ b/docs/guides/postgres/backup/stash/overview/index.md @@ -2,9 +2,9 @@ title: Backup & Restore PostgreSQL Using Stash menu: docs_{{ .version }}: - identifier: guides-pg-backup-overview + identifier: guides-pg-backup-overview-stashv1 name: Overview - parent: guides-pg-backup + parent: guides-pg-backup-stashv1 weight: 10 menu_name: docs_{{ .version }} section_menu_id: guides @@ -28,7 +28,7 @@ KubeDB uses [Stash](https://stash.run) to backup and restore databases. Stash by The following diagram shows how Stash takes backup of a PostgreSQL database. Open the image in a new tab to see the enlarged version.
-  PostgreSQL Backup Overview +  PostgreSQL Backup Overview
Fig: PostgreSQL Backup Overview
@@ -63,7 +63,7 @@ The backup process consists of the following steps: The following diagram shows how Stash restores backed up data into a PostgreSQL database. Open the image in a new tab to see the enlarged version.
-  Database Restore Overview +  Database Restore Overview
Fig: PostgreSQL Restore Process Overview
@@ -85,5 +85,5 @@ The restore process consists of the following steps: ## Next Steps -- Backup a standalone PostgreSQL database using Stash following the guide from [here](/docs/guides/postgres/backup/standalone/index.md). -- Configure a generic backup template for all the PostgreSQL databases of your cluster using Stash Auto-backup by following the guide from [here](/docs/guides/postgres/backup/auto-backup/index.md). +- Backup a standalone PostgreSQL database using Stash following the guide from [here](/docs/guides/postgres/backup/stash/standalone/index.md). +- Configure a generic backup template for all the PostgreSQL databases of your cluster using Stash Auto-backup by following the guide from [here](/docs/guides/postgres/backup/stash/auto-backup/index.md). diff --git a/docs/guides/postgres/backup/standalone/examples/appbinding.yaml b/docs/guides/postgres/backup/stash/standalone/examples/appbinding.yaml similarity index 100% rename from docs/guides/postgres/backup/standalone/examples/appbinding.yaml rename to docs/guides/postgres/backup/stash/standalone/examples/appbinding.yaml diff --git a/docs/guides/postgres/backup/standalone/examples/backupconfiguration.yaml b/docs/guides/postgres/backup/stash/standalone/examples/backupconfiguration.yaml similarity index 100% rename from docs/guides/postgres/backup/standalone/examples/backupconfiguration.yaml rename to docs/guides/postgres/backup/stash/standalone/examples/backupconfiguration.yaml diff --git a/docs/guides/postgres/backup/standalone/examples/postgres.yaml b/docs/guides/postgres/backup/stash/standalone/examples/postgres.yaml similarity index 100% rename from docs/guides/postgres/backup/standalone/examples/postgres.yaml rename to docs/guides/postgres/backup/stash/standalone/examples/postgres.yaml diff --git a/docs/guides/postgres/backup/standalone/examples/repository.yaml b/docs/guides/postgres/backup/stash/standalone/examples/repository.yaml similarity index 100% rename from docs/guides/postgres/backup/standalone/examples/repository.yaml rename to docs/guides/postgres/backup/stash/standalone/examples/repository.yaml diff --git a/docs/guides/postgres/backup/standalone/examples/restored-postgres.yaml b/docs/guides/postgres/backup/stash/standalone/examples/restored-postgres.yaml similarity index 100% rename from docs/guides/postgres/backup/standalone/examples/restored-postgres.yaml rename to docs/guides/postgres/backup/stash/standalone/examples/restored-postgres.yaml diff --git a/docs/guides/postgres/backup/standalone/examples/restoresession.yaml b/docs/guides/postgres/backup/stash/standalone/examples/restoresession.yaml similarity index 100% rename from docs/guides/postgres/backup/standalone/examples/restoresession.yaml rename to docs/guides/postgres/backup/stash/standalone/examples/restoresession.yaml diff --git a/docs/guides/postgres/backup/standalone/images/sample-postgres-backup.png b/docs/guides/postgres/backup/stash/standalone/images/sample-postgres-backup.png similarity index 100% rename from docs/guides/postgres/backup/standalone/images/sample-postgres-backup.png rename to docs/guides/postgres/backup/stash/standalone/images/sample-postgres-backup.png diff --git a/docs/guides/postgres/backup/standalone/index.md b/docs/guides/postgres/backup/stash/standalone/index.md similarity index 97% rename from docs/guides/postgres/backup/standalone/index.md rename to docs/guides/postgres/backup/stash/standalone/index.md index 4a146a1845..46ed3c04dc 100644 --- a/docs/guides/postgres/backup/standalone/index.md +++ b/docs/guides/postgres/backup/stash/standalone/index.md @@ -3,9 +3,9 @@ title: PostgreSQL | Stash description: Backup and restore standalone PostgreSQL database using Stash menu: docs_{{ .version }}: - identifier: guides-pg-backup-standalone + identifier: guides-pg-backup-standalone-stashv1 name: Standalone PostgreSQL - parent: guides-pg-backup + parent: guides-pg-backup-stashv1 weight: 20 menu_name: docs_{{ .version }} section_menu_id: guides @@ -22,7 +22,7 @@ Stash 0.9.0+ supports backup and restoration of PostgreSQL databases. This guide - Install KubeDB in your cluster following the steps [here](/docs/setup/README.md). - Install Stash in your cluster following the steps [here](https://stash.run/docs/latest/setup/install/stash/). - Install Stash `kubectl` plugin following the steps [here](https://stash.run/docs/latest/setup/install/kubectl-plugin/). -- If you are not familiar with how Stash backup and restore PostgreSQL databases, please check the following guide [here](/docs/guides/postgres/backup/overview/index.md): +- If you are not familiar with how Stash backup and restore PostgreSQL databases, please check the following guide [here](/docs/guides/postgres/backup/stash/overview/index.md): You have to be familiar with following custom resources: @@ -73,7 +73,7 @@ spec: Create the above `Postgres` crd, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/standalone/examples/postgres.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/standalone/examples/postgres.yaml postgres.kubedb.com/sample-postgres created ``` @@ -275,7 +275,7 @@ spec: Let's create the `Repository` we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/standalone/examples/repository.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/standalone/examples/repository.yaml repository.stash.appscode.com/gcs-repo created ``` @@ -320,7 +320,7 @@ Here, Let's create the `BackupConfiguration` object we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/standalone/examples/backupconfiguration.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/standalone/examples/backupconfiguration.yaml backupconfiguration.stash.appscode.com/sample-postgres-backup created ``` @@ -374,7 +374,7 @@ gcs-repo true 1.770 KiB 1 2m 4m1 Now, if we navigate to the GCS bucket, we are going to see backed up data has been stored in `demo/postgres/sample-postgres` directory as specified by `spec.backend.gcs.prefix` field of Repository crd.
- Backup data in GCS Bucket + Backup data in GCS Bucket
Fig: Backup data in GCS Bucket
@@ -443,7 +443,7 @@ Notice the `init` section. Here, we have specified `waitForInitialRestore: true` Let's create the above database, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/standalone/examples/restored-postgres.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/standalone/examples/restored-postgres.yaml postgres.kubedb.com/restored-postgres created ``` @@ -512,7 +512,7 @@ Here, Let's create the `RestoreSession` crd we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/standalone/examples/restoresession.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/stash/standalone/examples/restoresession.yaml restoresession.stash.appscode.com/sample-postgres-restore created ``` diff --git a/docs/guides/postgres/configuration/using-config-file.md b/docs/guides/postgres/configuration/using-config-file.md index 07f3a009b3..bda13136ed 100644 --- a/docs/guides/postgres/configuration/using-config-file.md +++ b/docs/guides/postgres/configuration/using-config-file.md @@ -203,7 +203,7 @@ If you would like to uninstall KubeDB operator, please follow the steps [here](/ ## Next Steps -- Learn about [backup and restore](/docs/guides/postgres/backup/overview/index.md) PostgreSQL database using Stash. +- Learn about [backup and restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL database using Stash. - Learn about initializing [PostgreSQL with Script](/docs/guides/postgres/initialization/script_source.md). - Want to setup PostgreSQL cluster? Check how to [configure Highly Available PostgreSQL Cluster](/docs/guides/postgres/clustering/ha_cluster.md) - Monitor your PostgreSQL database with KubeDB using [built-in Prometheus](/docs/guides/postgres/monitoring/using-builtin-prometheus.md). diff --git a/docs/guides/postgres/custom-rbac/using-custom-rbac.md b/docs/guides/postgres/custom-rbac/using-custom-rbac.md index eb4154ee50..fafb9db3d9 100644 --- a/docs/guides/postgres/custom-rbac/using-custom-rbac.md +++ b/docs/guides/postgres/custom-rbac/using-custom-rbac.md @@ -386,7 +386,7 @@ If you would like to uninstall the KubeDB operator, please follow the steps [her ## Next Steps -- Learn about [backup & restore](/docs/guides/postgres/backup/overview/index.md) of PostgreSQL databases using Stash. +- Learn about [backup & restore](/docs/guides/postgres/backup/stash/overview/index.md) of PostgreSQL databases using Stash. - Learn about initializing [PostgreSQL with Script](/docs/guides/postgres/initialization/script_source.md). - Want to setup PostgreSQL cluster? Check how to [configure Highly Available PostgreSQL Cluster](/docs/guides/postgres/clustering/ha_cluster.md) - Monitor your PostgreSQL instance with KubeDB using [built-in Prometheus](/docs/guides/postgres/monitoring/using-builtin-prometheus.md). diff --git a/docs/guides/postgres/initialization/script_source.md b/docs/guides/postgres/initialization/script_source.md index 8c2f26e881..fdd1b0b945 100644 --- a/docs/guides/postgres/initialization/script_source.md +++ b/docs/guides/postgres/initialization/script_source.md @@ -236,6 +236,6 @@ $ kubectl delete ns demo ## Next Steps -- Learn about [backup and restore](/docs/guides/postgres/backup/overview/index.md) PostgreSQL database using Stash. +- Learn about [backup and restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL database using Stash. - Want to setup PostgreSQL cluster? Check how to [configure Highly Available PostgreSQL Cluster](/docs/guides/postgres/clustering/ha_cluster.md) - Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/postgres/monitoring/using-builtin-prometheus.md b/docs/guides/postgres/monitoring/using-builtin-prometheus.md index 5c136b3f50..6adfa9f974 100644 --- a/docs/guides/postgres/monitoring/using-builtin-prometheus.md +++ b/docs/guides/postgres/monitoring/using-builtin-prometheus.md @@ -352,7 +352,7 @@ $ kubectl delete ns monitoring ## Next Steps -- Learn about [backup and restore](/docs/guides/postgres/backup/overview/index.md) PostgreSQL databases using Stash. +- Learn about [backup and restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL databases using Stash. - Monitor your PostgreSQL database with KubeDB using [`out-of-the-box` Prometheus operator](/docs/guides/postgres/monitoring/using-prometheus-operator.md). - Use [private Docker registry](/docs/guides/postgres/private-registry/using-private-registry.md) to deploy PostgreSQL with KubeDB. - Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/postgres/pitr/archiver.md b/docs/guides/postgres/pitr/archiver.md index c045693957..449f6ae024 100644 --- a/docs/guides/postgres/pitr/archiver.md +++ b/docs/guides/postgres/pitr/archiver.md @@ -423,7 +423,7 @@ $ kubectl delete ns demo ## Next Steps -- Learn about [backup and restore](/docs/guides/postgres/backup/overview/index.md) PostgreSQL database using Stash. +- Learn about [backup and restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL database using Stash. - Learn about initializing [PostgreSQL with Script](/docs/guides/postgres/initialization/script_source.md). - Learn about [custom PostgresVersions](/docs/guides/postgres/custom-versions/setup.md). - Want to setup PostgreSQL cluster? Check how to [configure Highly Available PostgreSQL Cluster](/docs/guides/postgres/clustering/ha_cluster.md) diff --git a/docs/guides/postgres/private-registry/using-private-registry.md b/docs/guides/postgres/private-registry/using-private-registry.md index 8b0ca9c417..8720a71804 100644 --- a/docs/guides/postgres/private-registry/using-private-registry.md +++ b/docs/guides/postgres/private-registry/using-private-registry.md @@ -202,7 +202,7 @@ If you would like to uninstall KubeDB operator, please follow the steps [here](/ ## Next Steps -- Learn about [backup and restore](/docs/guides/postgres/backup/overview/index.md) PostgreSQL database using Stash. +- Learn about [backup and restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL database using Stash. - Learn about initializing [PostgreSQL with Script](/docs/guides/postgres/initialization/script_source.md). - Want to setup PostgreSQL cluster? Check how to [configure Highly Available PostgreSQL Cluster](/docs/guides/postgres/clustering/ha_cluster.md) - Monitor your PostgreSQL database with KubeDB using [built-in Prometheus](/docs/guides/postgres/monitoring/using-builtin-prometheus.md). diff --git a/docs/guides/postgres/quickstart/quickstart.md b/docs/guides/postgres/quickstart/quickstart.md index 6d8aa1aff4..9a246e15af 100644 --- a/docs/guides/postgres/quickstart/quickstart.md +++ b/docs/guides/postgres/quickstart/quickstart.md @@ -541,7 +541,7 @@ If you are just testing some basic functionalities, you might want to avoid addi ## Next Steps -- Learn about [backup and restore](/docs/guides/postgres/backup/overview/index.md) PostgreSQL database using Stash. +- Learn about [backup and restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL database using Stash. - Learn about initializing [PostgreSQL with Script](/docs/guides/postgres/initialization/script_source.md). - Learn about [custom PostgresVersions](/docs/guides/postgres/custom-versions/setup.md). - Want to setup PostgreSQL cluster? Check how to [configure Highly Available PostgreSQL Cluster](/docs/guides/postgres/clustering/ha_cluster.md) diff --git a/docs/guides/postgres/remote-replica/remotereplica.md b/docs/guides/postgres/remote-replica/remotereplica.md index e4fc366685..33170ce789 100644 --- a/docs/guides/postgres/remote-replica/remotereplica.md +++ b/docs/guides/postgres/remote-replica/remotereplica.md @@ -355,7 +355,7 @@ kubectl delete ns demo ## Next Steps -- Learn about [backup and restore](/docs/guides/postgres/backup/overview/index.md) PostgreSQL database using Stash. +- Learn about [backup and restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL database using Stash. - Learn about initializing [PostgreSQL with Script](/docs/guides/postgres/initialization/script_source.md). - Learn about [custom PostgresVersions](/docs/guides/postgres/custom-versions/setup.md). - Want to setup PostgreSQL cluster? Check how to [configure Highly Available PostgreSQL Cluster](/docs/guides/postgres/clustering/ha_cluster.md) diff --git a/docs/guides/postgres/synchronous/index.md b/docs/guides/postgres/synchronous/index.md index 13fd957d48..4495e3712b 100644 --- a/docs/guides/postgres/synchronous/index.md +++ b/docs/guides/postgres/synchronous/index.md @@ -131,7 +131,7 @@ If you would like to uninstall KubeDB operator, please follow the steps [here](/ ## Next Steps -- Learn about [backup and restore](/docs/guides/postgres/backup/overview/index.md) PostgreSQL database using Stash. +- Learn about [backup and restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL database using Stash. - Learn about initializing [PostgreSQL with Script](/docs/guides/postgres/initialization/script_source.md). - Monitor your PostgreSQL database with KubeDB using [built-in Prometheus](/docs/guides/postgres/monitoring/using-builtin-prometheus.md). - Monitor your PostgreSQL database with KubeDB using [Prometheus operator](/docs/guides/postgres/monitoring/using-prometheus-operator.md). From 1d9a2269df0e27b9e533ad4c98aa0c54076391f2 Mon Sep 17 00:00:00 2001 From: Anisur Rahman Date: Fri, 13 Sep 2024 16:55:34 +0600 Subject: [PATCH 03/10] Incorporate with review changes Signed-off-by: Anisur Rahman --- .../kubestash/application-level/index.md | 30 +++++++++++++-- .../backup/kubestash/auto-backup/index.md | 18 +++------ .../backup/kubestash/customization/index.md | 31 ++++++++++++++- .../backup/kubestash/logical/index.md | 38 ++++++++++++++----- .../backup/kubestash/overview/index.md | 2 +- 5 files changed, 91 insertions(+), 28 deletions(-) diff --git a/docs/guides/postgres/backup/kubestash/application-level/index.md b/docs/guides/postgres/backup/kubestash/application-level/index.md index 21a249984f..0a355c4f5e 100644 --- a/docs/guides/postgres/backup/kubestash/application-level/index.md +++ b/docs/guides/postgres/backup/kubestash/application-level/index.md @@ -259,6 +259,19 @@ demo=# \d public | company | table | postgres (1 row) +# insert multiple rows of data into the table +demo=# INSERT INTO COMPANY (NAME, EMPLOYEE) VALUES ('TechCorp', 100), ('InnovateInc', 150), ('AlphaTech', 200); +INSERT 0 3 + +# verify the data insertion +demo=# SELECT * FROM COMPANY; + name | employee +-------------+---------- + TechCorp | 100 + InnovateInc | 150 + AlphaTech | 200 +(3 rows) + # quit from the database demo=# \q @@ -485,7 +498,6 @@ gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725449400 gcs-postgr ``` > Note: KubeStash creates a `Snapshot` with the following labels: -> - `kubedb.com/db-version: ` > - `kubestash.com/app-ref-kind: ` > - `kubestash.com/app-ref-name: ` > - `kubestash.com/app-ref-namespace: ` @@ -508,12 +520,13 @@ metadata: - kubestash.com/cleanup generation: 1 labels: - kubedb.com/db-version: "16.1" kubestash.com/app-ref-kind: Postgres kubestash.com/app-ref-name: sample-postgres kubestash.com/app-ref-namespace: demo kubestash.com/repo-name: gcs-postgres-repo - name: gcs-postgres-repo-sample-postgreckup-frequent-backup-1725449400 + annotations: + kubedb.com/db-version: "16.1" + name: gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725449400 namespace: demo ownerReferences: - apiVersion: storage.kubestash.com/v1alpha1 @@ -581,7 +594,7 @@ status: totalComponents: 2 ``` -> KubeStash uses the `pg_dump` command to take backups of target PostgreSQL databases. Therefore, the component name for `logical backups` is set as `dump`. +> KubeStash uses a logical backup approach to take backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Do the same for auto-backup, application backup and customize backup if necessary. > KubeStash set component name as `manifest` for the `manifest backup` of PostgreSQL databases. @@ -729,6 +742,15 @@ demo=# \d public | company | table | postgres (1 row) +# Verify that the sample data has been restored +demo=# SELECT * FROM COMPANY; + name | employee +-------------+---------- + TechCorp | 100 + InnovateInc | 150 + AlphaTech | 200 +(3 rows) + # disconnect from the database demo=# \q diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/index.md b/docs/guides/postgres/backup/kubestash/auto-backup/index.md index a93e780858..882dba838b 100644 --- a/docs/guides/postgres/backup/kubestash/auto-backup/index.md +++ b/docs/guides/postgres/backup/kubestash/auto-backup/index.md @@ -17,14 +17,13 @@ KubeStash can automatically be configured to backup any `PostgreSQL` databases i In this tutorial, we are going to show how you can configure a backup blueprint for `PostgreSQL` databases in your cluster and backup them with a few annotations. - ## Before You Begin - At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. - Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). - Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). - Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). -- If you are not familiar with how KubeStash backup and restore PostgreSQL databases, please check the following guide [here](/docs/guides/postgres/backup/kubestash/overview/index.md). +- If you are not familiar with how KubeStash backup and restore `PostgreSQL` databases, please check the following guide [here](/docs/guides/postgres/backup/kubestash/overview/index.md). You should be familiar with the following `KubeStash` concepts: @@ -45,7 +44,6 @@ namespace/demo created > **Note:** YAML files used in this tutorial are stored in [docs/guides/postgres/backup/kubestash/auto-backup/examples](https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/auto-backup/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. - ### Prepare Backend We are going to store our backed up data into a `GCS` bucket. We have to create a `Secret` with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). @@ -162,7 +160,6 @@ spec: from: All backupConfigurationTemplate: deletionPolicy: OnDelete - # ============== Blueprint for Backends of BackupConfiguration ================= backends: - name: gcs-backend storageRef: @@ -171,7 +168,6 @@ spec: retentionPolicy: name: demo-retention namespace: demo - # ============== Blueprint for Sessions of BackupConfiguration ================= sessions: - name: frequent-backup sessionHistoryLimit: 3 @@ -204,7 +200,7 @@ $ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" > backupblueprint.core.kubestash.com/postgres-default-backup-blueprint created ``` -Now, we are ready to backup our `PostgreSQL` databases using few annotations. You can check available auto-backup annotations for a databases from [here](https://kubestash.com/docs/latest/concepts/crds/backupblueprint/). +Now, we are ready to backup our `PostgreSQL` databases using few annotations. **Create Database** @@ -395,7 +391,6 @@ default-blueprint-appbinding-samgres-frequent-backup-1725533628 default-bluepr ``` > Note: KubeStash creates a `Snapshot` with the following labels: -> - `kubedb.com/db-version: ` > - `kubestash.com/app-ref-kind: ` > - `kubestash.com/app-ref-name: ` > - `kubestash.com/app-ref-namespace: ` @@ -418,11 +413,12 @@ metadata: - kubestash.com/cleanup generation: 1 labels: - kubedb.com/db-version: "16.1" kubestash.com/app-ref-kind: Postgres kubestash.com/app-ref-name: sample-postgres kubestash.com/app-ref-namespace: demo kubestash.com/repo-name: default-blueprint + annotations: + kubedb.com/db-version: "16.1" name: default-blueprint-appbinding-samgres-frequent-backup-1725533628 namespace: demo ownerReferences: @@ -479,7 +475,7 @@ status: totalComponents: 1 ``` -> KubeStash uses the `pg_dump` command to take backups of target PostgreSQL databases. Therefore, the component name for `logical backups` is set as `dump`. +> KubeStash uses a logical backup approach to take backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Do the same for auto-backup, application backup and customize backup if necessary. Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `blueprint/default-blueprint/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/default-blueprint/snapshots` directory. @@ -561,7 +557,6 @@ backupblueprint.core.kubestash.com/postgres-customize-backup-blueprint created Now, we are ready to backup our `PostgreSQL` databases using few annotations. You can check available auto-backup annotations for a databases from [here](https://kubestash.com/docs/latest/concepts/crds/backupblueprint/). - **Create Database** Now, we are going to create an `PostgreSQL` CR in demo namespace. @@ -736,7 +731,6 @@ appbinding-sample-postgres-frequent-backup-1725597000 BackupConfiguration We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. - **Verify Backup:** Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `customize-blueprint` has been updated by the following command, @@ -841,7 +835,7 @@ status: ``` -> KubeStash uses the `pg_dump` command to take backups of target PostgreSQL databases. Therefore, the component name for `logical backups` is set as `dump`. +> KubeStash uses a logical backup approach to take backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Do the same for auto-backup, application backup and customize backup if necessary. Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `blueprint/demo/sample-postgres-2/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/demo/sample-postgres-2/snapshots` directory. diff --git a/docs/guides/postgres/backup/kubestash/customization/index.md b/docs/guides/postgres/backup/kubestash/customization/index.md index ef91b9ded2..a904f89d4d 100644 --- a/docs/guides/postgres/backup/kubestash/customization/index.md +++ b/docs/guides/postgres/backup/kubestash/customization/index.md @@ -117,7 +117,6 @@ spec: > **WARNING**: Make sure that your provided database has been created before taking backup. - ### Using multiple repositories You can configure multiple repositories for the same backend. For example, if you want to back up the `/postgres` directory using the `gcs-postgres-repo` repository, you can also back up another directory, such as `/postgres-copy`, by using a different repository, like `gcs-postgres-repo-2`. @@ -273,6 +272,35 @@ spec: `KubeStash` uses [psql](https://www.postgresql.org/docs/current/app-psql.html) during the restore process. In this section, we are going to show how you can pass arguments to the restore process, restore a specific snapshot, run restore job as a specific user, etc. +### Passing arguments to the restore process + +You can pass any supported `psql` arguments to the restore process using the `args` field within the `addon.tasks[*].params` section. This example demonstrates how to specify a database `testdb` to connect to during the restore process. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup-restore + params: + args: --dbname=testdb +``` ### Restore specific snapshot @@ -313,7 +341,6 @@ spec: - name: logical-backup-restore ``` - ### Running restore job as a specific user Similar to the backup process under the `addon.jobTemplate.spec.` you can provide `securityContext` to run the restore job as a specific user. diff --git a/docs/guides/postgres/backup/kubestash/logical/index.md b/docs/guides/postgres/backup/kubestash/logical/index.md index a0d0060b60..2a86b9a401 100644 --- a/docs/guides/postgres/backup/kubestash/logical/index.md +++ b/docs/guides/postgres/backup/kubestash/logical/index.md @@ -263,6 +263,19 @@ demo=# \d public | company | table | postgres (1 row) +# insert multiple rows of data into the table +demo=# INSERT INTO COMPANY (NAME, EMPLOYEE) VALUES ('TechCorp', 100), ('InnovateInc', 150), ('AlphaTech', 200); +INSERT 0 3 + +# verify the data insertion +demo=# SELECT * FROM COMPANY; + name | employee +-------------+---------- + TechCorp | 100 + InnovateInc | 150 + AlphaTech | 200 +(3 rows) + # quit from the database demo=# \q @@ -484,7 +497,6 @@ gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725449400 gcs-postgr ``` > Note: KubeStash creates a `Snapshot` with the following labels: -> - `kubedb.com/db-version: ` > - `kubestash.com/app-ref-kind: ` > - `kubestash.com/app-ref-name: ` > - `kubestash.com/app-ref-namespace: ` @@ -507,12 +519,13 @@ metadata: - kubestash.com/cleanup generation: 1 labels: - kubedb.com/db-version: "16.1" kubestash.com/app-ref-kind: Postgres kubestash.com/app-ref-name: sample-postgres kubestash.com/app-ref-namespace: demo kubestash.com/repo-name: gcs-postgres-repo - name: gcs-postgres-repo-sample-postgreckup-frequent-backup-1725449400 + annotations: + kubedb.com/db-version: "16.1" + name: gcs-postgres-repo-sample-postgres-backup-frequent-backup-1725449400 namespace: demo ownerReferences: - apiVersion: storage.kubestash.com/v1alpha1 @@ -568,13 +581,12 @@ status: totalComponents: 1 ``` -> KubeStash uses the `pg_dump` command to take backups of target PostgreSQL databases. Therefore, the component name for `logical backups` is set as `dump`. +> KubeStash uses a logical backup approach to take backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Do the same for auto-backup, application backup and customize backup if necessary. Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/popstgres/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/postgres/snapshots` directory. > Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. - ## Restore In this section, we are going to restore the database from the backup we have taken in the previous section. We are going to deploy a new database and initialize it from the backup. @@ -619,7 +631,7 @@ If you check the database status, you will see it is stuck in **`Provisioning`** ```bash $ kubectl get postgres -n demo restored-postgres -NAME VERSION STATUS AGE +NAME VERSION STATUS AGE restored-postgres 8.2.0 Provisioning 61s ``` @@ -662,7 +674,7 @@ Here, Let's create the RestoreSession CRD object we have shown above, ```bash -$ kubectl apply -f **https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples/restoresession.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/backup/kubestash/logical/examples/restoresession.yaml restoresession.core.kubestash.com/sample-postgres-restore created ``` @@ -734,6 +746,15 @@ demo=# \d public | company | table | postgres (1 row) +# Verify that the sample data has been restored +demo=# SELECT * FROM COMPANY; + name | employee +-------------+---------- + TechCorp | 100 + InnovateInc | 150 + AlphaTech | 200 +(3 rows) + # disconnect from the database demo=# \q @@ -743,7 +764,6 @@ demo=# \q So, from the above output, we can see the `demo` database we had created in the original database `sample-postgres` has been restored in the `restored-postgres` database. - ## Cleanup To cleanup the Kubernetes resources created by this tutorial, run: @@ -751,10 +771,10 @@ To cleanup the Kubernetes resources created by this tutorial, run: ```bash kubectl delete backupconfigurations.core.kubestash.com -n demo sample-postgres-backup kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-postgres +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention kubectl delete backupstorage -n demo gcs-storage kubectl delete secret -n demo gcs-secret kubectl delete secret -n demo encrypt-secret -kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention kubectl delete postgres -n demo restored-postgres kubectl delete postgres -n demo sample-postgres ``` \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/overview/index.md b/docs/guides/postgres/backup/kubestash/overview/index.md index 8d82725ef2..dd56b25368 100644 --- a/docs/guides/postgres/backup/kubestash/overview/index.md +++ b/docs/guides/postgres/backup/kubestash/overview/index.md @@ -91,7 +91,7 @@ The restore process consists of the following steps: 7. Then, the `Job` downloads the backed up data from the backend and injects into the desired database. KubeStash pipes the downloaded data to the respective database tool to inject into the database. Hence, restore `Job` does not require a large volume to download entire backup data inside it. -8. Finally, when the restore process is completed, the `Job` updates the `status.components[dump]` field of the `RestoreSession` with restore information of the target database. +8. Finally, when the restore process is completed, the `Job` updates the `status.components[*]` field of the `RestoreSession` with restore information of the target database. ## Next Steps From 63f611be1ad2aa694ae2ffdf69f3a6a21469336e Mon Sep 17 00:00:00 2001 From: Anisur Rahman Date: Mon, 16 Sep 2024 16:13:09 +0600 Subject: [PATCH 04/10] Resolve saiem bhai's review changes Signed-off-by: Anisur Rahman --- .../examples/backupstorage.yaml | 2 +- .../kubestash/application-level/index.md | 7 +++--- .../auto-backup/examples/backupstorage.yaml | 2 +- .../examples/customize-backupblueprint.yaml | 2 -- .../examples/default-backupblueprint.yaml | 2 -- .../backup/kubestash/auto-backup/index.md | 11 +++------ ...repository.yaml => multiple-backends.yaml} | 11 +++++++-- ...kupstorage.yaml => gcs-backupstorage.yaml} | 2 +- .../common/s3-backupstorage.yaml | 19 +++++++++++++++ .../backup/kubestash/customization/index.md | 17 ++++++++++---- .../customization/restore/passing-args.yaml | 23 +++++++++++++++++++ .../logical/examples/backupstorage.yaml | 2 +- .../backup/kubestash/logical/index.md | 4 ++-- .../backup/kubestash/overview/index.md | 12 +++++----- 14 files changed, 81 insertions(+), 35 deletions(-) rename docs/guides/postgres/backup/kubestash/customization/backup/{multiple-repository.yaml => multiple-backends.yaml} (80%) rename docs/guides/postgres/backup/kubestash/customization/common/{backupstorage.yaml => gcs-backupstorage.yaml} (94%) create mode 100644 docs/guides/postgres/backup/kubestash/customization/common/s3-backupstorage.yaml create mode 100644 docs/guides/postgres/backup/kubestash/customization/restore/passing-args.yaml diff --git a/docs/guides/postgres/backup/kubestash/application-level/examples/backupstorage.yaml b/docs/guides/postgres/backup/kubestash/application-level/examples/backupstorage.yaml index 0461b26762..6ab3df02ac 100644 --- a/docs/guides/postgres/backup/kubestash/application-level/examples/backupstorage.yaml +++ b/docs/guides/postgres/backup/kubestash/application-level/examples/backupstorage.yaml @@ -14,4 +14,4 @@ spec: allowedNamespaces: from: All default: true - deletionPolicy: WipeOut \ No newline at end of file + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/application-level/index.md b/docs/guides/postgres/backup/kubestash/application-level/index.md index 0a355c4f5e..404fa34eec 100644 --- a/docs/guides/postgres/backup/kubestash/application-level/index.md +++ b/docs/guides/postgres/backup/kubestash/application-level/index.md @@ -319,7 +319,7 @@ spec: allowedNamespaces: from: All default: true - deletionPolicy: WipeOut + deletionPolicy: Delete ``` Let's create the BackupStorage we have shown above, @@ -425,8 +425,7 @@ spec: - `.spec.sessions[*].schedule` specifies that we want to backup at `5 minutes` interval. - `.spec.target` refers to the targeted `sample-postgres` PostgreSQL database that we created earlier. -- `.spec.sessions[*].addon.tasks[*].name[*]` specifies that both the `manifest-backup` and `logical-backup` will be taken together. - +- `.spec.sessions[*].addon.tasks[*].name[*]` specifies that both the `manifest-backup` and `logical-backup` tasks will be executed. Let's create the `BackupConfiguration` CR that we have shown above, @@ -594,7 +593,7 @@ status: totalComponents: 2 ``` -> KubeStash uses a logical backup approach to take backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Do the same for auto-backup, application backup and customize backup if necessary. +> KubeStash uses `pg_dump` or `pg_dumpall` to perform backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. > KubeStash set component name as `manifest` for the `manifest backup` of PostgreSQL databases. diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/examples/backupstorage.yaml b/docs/guides/postgres/backup/kubestash/auto-backup/examples/backupstorage.yaml index 0461b26762..6ab3df02ac 100644 --- a/docs/guides/postgres/backup/kubestash/auto-backup/examples/backupstorage.yaml +++ b/docs/guides/postgres/backup/kubestash/auto-backup/examples/backupstorage.yaml @@ -14,4 +14,4 @@ spec: allowedNamespaces: from: All default: true - deletionPolicy: WipeOut \ No newline at end of file + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml b/docs/guides/postgres/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml index ea1a2eceda..fd729c3aba 100644 --- a/docs/guides/postgres/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml +++ b/docs/guides/postgres/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml @@ -9,7 +9,6 @@ spec: from: All backupConfigurationTemplate: deletionPolicy: OnDelete - # ============== Blueprint for Backends of BackupConfiguration ================= backends: - name: gcs-backend storageRef: @@ -18,7 +17,6 @@ spec: retentionPolicy: name: demo-retention namespace: demo - # ============== Blueprint for Sessions of BackupConfiguration ================= sessions: - name: frequent-backup sessionHistoryLimit: 3 diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml b/docs/guides/postgres/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml index 627cdfe5a0..5fbafc0f08 100644 --- a/docs/guides/postgres/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml +++ b/docs/guides/postgres/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml @@ -9,7 +9,6 @@ spec: from: All backupConfigurationTemplate: deletionPolicy: OnDelete - # ============== Blueprint for Backends of BackupConfiguration ================= backends: - name: gcs-backend storageRef: @@ -18,7 +17,6 @@ spec: retentionPolicy: name: demo-retention namespace: demo - # ============== Blueprint for Sessions of BackupConfiguration ================= sessions: - name: frequent-backup sessionHistoryLimit: 3 diff --git a/docs/guides/postgres/backup/kubestash/auto-backup/index.md b/docs/guides/postgres/backup/kubestash/auto-backup/index.md index 882dba838b..56861e80b2 100644 --- a/docs/guides/postgres/backup/kubestash/auto-backup/index.md +++ b/docs/guides/postgres/backup/kubestash/auto-backup/index.md @@ -82,7 +82,7 @@ spec: allowedNamespaces: from: All default: true - deletionPolicy: WipeOut + deletionPolicy: Delete ``` Let's create the BackupStorage we have shown above, @@ -359,7 +359,6 @@ status: Notice the `spec.backends`, `spec.sessions` and `spec.target` sections, KubeStash automatically resolved those info from the `BackupBluePrint` and created above `BackupConfiguration`. - **Verify BackupSession:** KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. @@ -475,13 +474,12 @@ status: totalComponents: 1 ``` -> KubeStash uses a logical backup approach to take backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Do the same for auto-backup, application backup and customize backup if necessary. +> KubeStash uses `pg_dump` or `pg_dumpall` to perform backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `blueprint/default-blueprint/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/default-blueprint/snapshots` directory. > Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. - ## Auto-backup with custom configurations In this section, we are going to backup a `PostgreSQL` database of `demo` namespace. We are going to use the custom configurations which will be specified in the `BackupBlueprint` CR. @@ -504,7 +502,6 @@ spec: from: All backupConfigurationTemplate: deletionPolicy: OnDelete - # ============== Blueprint for Backends of BackupConfiguration ================= backends: - name: gcs-backend storageRef: @@ -513,7 +510,6 @@ spec: retentionPolicy: name: demo-retention namespace: demo - # ============== Blueprint for Sessions of BackupConfiguration ================= sessions: - name: frequent-backup sessionHistoryLimit: 3 @@ -834,8 +830,7 @@ status: totalComponents: 1 ``` - -> KubeStash uses a logical backup approach to take backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Do the same for auto-backup, application backup and customize backup if necessary. +> KubeStash uses `pg_dump` or `pg_dumpall` to perform backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `blueprint/demo/sample-postgres-2/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/demo/sample-postgres-2/snapshots` directory. diff --git a/docs/guides/postgres/backup/kubestash/customization/backup/multiple-repository.yaml b/docs/guides/postgres/backup/kubestash/customization/backup/multiple-backends.yaml similarity index 80% rename from docs/guides/postgres/backup/kubestash/customization/backup/multiple-repository.yaml rename to docs/guides/postgres/backup/kubestash/customization/backup/multiple-backends.yaml index 25cffb0b05..422754ccf5 100644 --- a/docs/guides/postgres/backup/kubestash/customization/backup/multiple-repository.yaml +++ b/docs/guides/postgres/backup/kubestash/customization/backup/multiple-backends.yaml @@ -17,6 +17,13 @@ spec: retentionPolicy: name: demo-retention namespace: demo + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo sessions: - name: frequent-backup scheduler: @@ -30,8 +37,8 @@ spec: encryptionSecret: name: encrypt-secret namespace: demo - - name: gcs-postgres-repo-2 - backend: gcs-backend + - name: s3-postgres-repo + backend: s3-backend directory: /postgres-copy encryptionSecret: name: encrypt-secret diff --git a/docs/guides/postgres/backup/kubestash/customization/common/backupstorage.yaml b/docs/guides/postgres/backup/kubestash/customization/common/gcs-backupstorage.yaml similarity index 94% rename from docs/guides/postgres/backup/kubestash/customization/common/backupstorage.yaml rename to docs/guides/postgres/backup/kubestash/customization/common/gcs-backupstorage.yaml index 0461b26762..5972fd3a31 100644 --- a/docs/guides/postgres/backup/kubestash/customization/common/backupstorage.yaml +++ b/docs/guides/postgres/backup/kubestash/customization/common/gcs-backupstorage.yaml @@ -13,5 +13,5 @@ spec: usagePolicy: allowedNamespaces: from: All - default: true + default: false deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/common/s3-backupstorage.yaml b/docs/guides/postgres/backup/kubestash/customization/common/s3-backupstorage.yaml new file mode 100644 index 0000000000..c87d26f7ec --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/common/s3-backupstorage.yaml @@ -0,0 +1,19 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: s3-storage + namespace: demo +spec: + storage: + provider: s3 + s3: + bucket: kubestash + region: us-east-1 + endpoint: us-east-1.linodeobjects.com + secretName: s3-secret + prefix: sunny + usagePolicy: + allowedNamespaces: + from: All + default: false + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/customization/index.md b/docs/guides/postgres/backup/kubestash/customization/index.md index a904f89d4d..40cc2a3ddf 100644 --- a/docs/guides/postgres/backup/kubestash/customization/index.md +++ b/docs/guides/postgres/backup/kubestash/customization/index.md @@ -69,7 +69,7 @@ spec: ### Passing a target database to the backup process -KubeStash PostgreSQL addon uses the [pg_dumpall](https://www.postgresql.org/docs/current/app-pg-dumpall.html) command by default for backups. For a single database backup, you need to rewrite the dump command. You can do this by setting `backupCmd` to [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html) under the `addon.tasks[*].params` section and specifying the database name using the `args` parameter in the same section. +KubeStash PostgreSQL addon uses the [pg_dumpall](https://www.postgresql.org/docs/current/app-pg-dumpall.html) command by default for backups. If you want to back up a single database, you’ll need to switch the command to [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html). You can do this by setting `backupCmd` to `pg_dump` under the `addon.tasks[*].params` section and specifying the database name using the `args` parameter in the same section. The below example shows how you can set `pg_dump` and pass target database name during backup. @@ -117,9 +117,9 @@ spec: > **WARNING**: Make sure that your provided database has been created before taking backup. -### Using multiple repositories +### Using multiple backends -You can configure multiple repositories for the same backend. For example, if you want to back up the `/postgres` directory using the `gcs-postgres-repo` repository, you can also back up another directory, such as `/postgres-copy`, by using a different repository, like `gcs-postgres-repo-2`. +You can configure multiple backends within a single `backupConfiguration`. To back up the same data to different backends, such as S3 and GCS, declare each backend in the `.spe.backends` section. Then, reference these backends in the `.spec.sessions[*].repositories` section. ```yaml apiVersion: core.kubestash.com/v1alpha1 @@ -141,6 +141,13 @@ spec: retentionPolicy: name: demo-retention namespace: demo + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo sessions: - name: frequent-backup scheduler: @@ -154,8 +161,8 @@ spec: encryptionSecret: name: encrypt-secret namespace: demo - - name: gcs-postgres-repo-2 - backend: gcs-backend + - name: s3-postgres-repo + backend: s3-backend directory: /postgres-copy encryptionSecret: name: encrypt-secret diff --git a/docs/guides/postgres/backup/kubestash/customization/restore/passing-args.yaml b/docs/guides/postgres/backup/kubestash/customization/restore/passing-args.yaml new file mode 100644 index 0000000000..71cd9f9ebd --- /dev/null +++ b/docs/guides/postgres/backup/kubestash/customization/restore/passing-args.yaml @@ -0,0 +1,23 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: sample-postgres-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Postgres + namespace: demo + name: restored-postgres + dataSource: + repository: gcs-postgres-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: postgres-addon + tasks: + - name: logical-backup-restore + params: + args: --dbname=testdb \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml b/docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml index 0461b26762..6ab3df02ac 100644 --- a/docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml +++ b/docs/guides/postgres/backup/kubestash/logical/examples/backupstorage.yaml @@ -14,4 +14,4 @@ spec: allowedNamespaces: from: All default: true - deletionPolicy: WipeOut \ No newline at end of file + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/postgres/backup/kubestash/logical/index.md b/docs/guides/postgres/backup/kubestash/logical/index.md index 2a86b9a401..0d0a2031bf 100644 --- a/docs/guides/postgres/backup/kubestash/logical/index.md +++ b/docs/guides/postgres/backup/kubestash/logical/index.md @@ -323,7 +323,7 @@ spec: allowedNamespaces: from: All default: true - deletionPolicy: WipeOut + deletionPolicy: Delete ``` Let's create the BackupStorage we have shown above, @@ -581,7 +581,7 @@ status: totalComponents: 1 ``` -> KubeStash uses a logical backup approach to take backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Do the same for auto-backup, application backup and customize backup if necessary. +> KubeStash uses `pg_dump` or `pg_dumpall` to perform backups of target `PostgreSQL` databases. Therefore, the component name for logical backups is set as `dump`. Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/popstgres/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/postgres/snapshots` directory. diff --git a/docs/guides/postgres/backup/kubestash/overview/index.md b/docs/guides/postgres/backup/kubestash/overview/index.md index dd56b25368..4ed7970d59 100644 --- a/docs/guides/postgres/backup/kubestash/overview/index.md +++ b/docs/guides/postgres/backup/kubestash/overview/index.md @@ -38,9 +38,9 @@ The backup process consists of the following steps: 2. Then, she creates a `BackupStorage` custom resource that specifies the backend information, along with the `Secret` containing the credentials needed to access the backend. -3. KubeStash operator watches for `BackupStorage` custom resources. When it finds a `BackupStorage` object, it initializes the `BackupStorage` by uploading the `metadata.yaml` file into the target storage. +3. KubeStash operator watches for `BackupStorage` custom resources. When it finds a `BackupStorage` object, it initializes the `BackupStorage` by uploading the `metadata.yaml` file to the specified backend. -4. Then, she creates a `BackupConfiguration` custom resource that specifies the targeted the KubeDB managed `PostgreSQL` database, the `Addon` info with a specified task, etc. It also provides information about one or more repositories, each indicating a path and a `BackupStorage` for storing the backed-up data. +4. Next, she creates a `BackupConfiguration` custom resource that specifies the target database, addon information (including backup tasks), backup schedules, storage backends for storing the backup data, and other additional settings. 5. KubeStash operator watches for `BackupConfiguration` objects. @@ -50,7 +50,7 @@ The backup process consists of the following steps: 8. Then, it creates a `CronJob` for each session with the schedule specified in `BackupConfiguration` to trigger backup periodically. -9. KubeStash operator triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are triggered by the `CronJob` according to the specified schedule. +9. KubeStash operator triggers an instant backup as soon as the `BackupConfiguration` is ready. Backups are otherwise triggered by the `CronJob` based on the specified schedule. 10. KubeStash operator watches for `BackupSession` custom resources. @@ -60,7 +60,7 @@ The backup process consists of the following steps: 13. Then, it creates the `Job` to backup the targeted `PostgreSQL` database. -14. The backup `Job` reads necessary information (e.g. auth secret, port) to connect with the database from the `AppBinding` crd. It also reads backend information and access credentials from BackupStorage crd, Storage Secret and Repository path respectively. +14. The backup `Job` reads necessary information (e.g. auth secret, port) to connect with the database from the `AppBinding` CR. It also reads backend information and access credentials from `BackupStorage` CR, Storage Secret and `Repository` path respectively. 15. Then, the `Job` dumps the targeted `PostgreSQL` database and uploads the output to the backend. KubeStash pipes the output of dump command to uploading process. Hence, backup `Job` does not require a large volume to hold the entire dump output. @@ -79,7 +79,7 @@ The restore process consists of the following steps: 1. At first, a user creates a `PostgreSQL` database where the data will be restored or the user can use the same `PostgreSQL` database. -2. Then, she creates a `RestoreSession` custom resource that specifies the target `PostgreSQL` database where the backed-up data will be restored. the `Repository` object that points to a `BackupStorage` that holds backend information, and the target `Snapshot`, which will be restored. It also specifies the `Addon` info with task to use to restore. +2. Then, she creates a `RestoreSession` custom resource that specifies the target database where the backed-up data will be restored, addon information (including restore tasks), the target snapshot to be restored, the Repository containing that snapshot, and other additional settings. 3. KubeStash operator watches for `RestoreSession` custom resources. @@ -87,7 +87,7 @@ The restore process consists of the following steps: 5. Then, it creates the `Job` to restore the target. -6. The `Job` reads necessary information to connect with the database from respective `AppBinding` crd. It also reads backend information and access credentials from `Repository` crd and storage `Secret` respectively. +6. The `Job` reads necessary information to connect with the database from respective `AppBinding` CR. It also reads backend information and access credentials from `Repository` CR and storage `Secret` respectively. 7. Then, the `Job` downloads the backed up data from the backend and injects into the desired database. KubeStash pipes the downloaded data to the respective database tool to inject into the database. Hence, restore `Job` does not require a large volume to download entire backup data inside it. From cd2e43588dd2f2153432b68d97e18619fe2dae18 Mon Sep 17 00:00:00 2001 From: Ashraful Haque Tani Date: Tue, 17 Sep 2024 18:01:43 +0600 Subject: [PATCH 05/10] SingleStore Backup & Restore Signed-off-by: Ashraful Haque Tani --- docs/guides/singlestore/backup/_index.md | 10 + .../singlestore/backup/kubestash/_index.md | 10 + .../logical/examples/backupconfiguration.yaml | 36 + .../logical/examples/backupstorage.yaml | 17 + .../logical/examples/restoresession.yaml | 21 + .../logical/examples/retentionpolicy.yaml | 15 + .../logical/examples/sdb-sample.yaml | 50 ++ .../backup/kubestash/logical/index.md | 756 ++++++++++++++++++ .../overview/images/backup_overview.svg | 1 + .../overview/images/kubedb_plus_kubestash.svg | 21 + .../overview/images/restore_overview.svg | 1 + .../backup/kubestash/overview/index.md | 98 +++ 12 files changed, 1036 insertions(+) create mode 100644 docs/guides/singlestore/backup/_index.md create mode 100644 docs/guides/singlestore/backup/kubestash/_index.md create mode 100644 docs/guides/singlestore/backup/kubestash/logical/examples/backupconfiguration.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/logical/examples/backupstorage.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/logical/examples/restoresession.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/logical/examples/retentionpolicy.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/logical/examples/sdb-sample.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/logical/index.md create mode 100644 docs/guides/singlestore/backup/kubestash/overview/images/backup_overview.svg create mode 100644 docs/guides/singlestore/backup/kubestash/overview/images/kubedb_plus_kubestash.svg create mode 100644 docs/guides/singlestore/backup/kubestash/overview/images/restore_overview.svg create mode 100644 docs/guides/singlestore/backup/kubestash/overview/index.md diff --git a/docs/guides/singlestore/backup/_index.md b/docs/guides/singlestore/backup/_index.md new file mode 100644 index 0000000000..79fe93f9e6 --- /dev/null +++ b/docs/guides/singlestore/backup/_index.md @@ -0,0 +1,10 @@ +--- +title: Backup & Restore SingleStore +menu: + docs_{{ .version }}: + identifier: guides-sdb-backup + name: Back & Restore + parent: guides-singlestore + weight: 40 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/singlestore/backup/kubestash/_index.md b/docs/guides/singlestore/backup/kubestash/_index.md new file mode 100644 index 0000000000..566ec1e175 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/_index.md @@ -0,0 +1,10 @@ +--- +title: Backup & Restore SingleStore | KubeStash +menu: + docs_{{ .version }}: + identifier: guides-sdb-backup-stashv2 + name: KubeStash (aka Stash 2.0) + parent: guides-sdb-backup + weight: 50 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/singlestore/backup/kubestash/logical/examples/backupconfiguration.yaml b/docs/guides/singlestore/backup/kubestash/logical/examples/backupconfiguration.yaml new file mode 100644 index 0000000000..5434c03eac --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/logical/examples/backupconfiguration.yaml @@ -0,0 +1,36 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-sdb-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sdb-sample + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-sdb-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/logical/examples/backupstorage.yaml b/docs/guides/singlestore/backup/kubestash/logical/examples/backupstorage.yaml new file mode 100644 index 0000000000..6ab3df02ac --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/logical/examples/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/logical/examples/restoresession.yaml b/docs/guides/singlestore/backup/kubestash/logical/examples/restoresession.yaml new file mode 100644 index 0000000000..2438d4533a --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/logical/examples/restoresession.yaml @@ -0,0 +1,21 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore diff --git a/docs/guides/singlestore/backup/kubestash/logical/examples/retentionpolicy.yaml b/docs/guides/singlestore/backup/kubestash/logical/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/logical/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/logical/examples/sdb-sample.yaml b/docs/guides/singlestore/backup/kubestash/logical/examples/sdb-sample.yaml new file mode 100644 index 0000000000..f5123df643 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/logical/examples/sdb-sample.yaml @@ -0,0 +1,50 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: sdb-sample + namespace: demo +spec: + version: "8.7.10" + topology: + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + storageType: Durable + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/logical/index.md b/docs/guides/singlestore/backup/kubestash/logical/index.md new file mode 100644 index 0000000000..91de127770 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/logical/index.md @@ -0,0 +1,756 @@ +--- +title: Backup & Restore SingleStore | KubeStash +description: Backup SingleStore database using KubeStash +menu: + docs_{{ .version }}: + identifier: guides-sdb-backup-logical-backup-stashv2 + name: Logical Backup + parent: guides-sdb-backup-stashv2 + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Backup and Restore SingleStore database using KubeStash + +KubeStash allows you to backup and restore `SingleStore` databases. It supports backups for `SingleStore` instances running in Standalone, and cluster configurations. KubeStash makes managing your `SingleStore` backups and restorations more straightforward and efficient. + +This guide will give you how you can take backup and restore your `SingleStore` databases using `Kubestash`. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. +- Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). +- Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). +- Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- If you are not familiar with how KubeStash backup and restore MySQL databases, please check the following guide [here](/docs/guides/mysql/backup/kubestash/overview/index.md). + +You should be familiar with the following `KubeStash` concepts: + +- [BackupStorage](https://kubestash.com/docs/latest/concepts/crds/backupstorage/) +- [BackupConfiguration](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/) +- [BackupSession](https://kubestash.com/docs/latest/concepts/crds/backupsession/) +- [RestoreSession](https://kubestash.com/docs/latest/concepts/crds/restoresession/) +- [Addon](https://kubestash.com/docs/latest/concepts/crds/addon/) +- [Function](https://kubestash.com/docs/latest/concepts/crds/function/) +- [Task](https://kubestash.com/docs/latest/concepts/crds/addon/#task-specification) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/guides/mysql/backup/kubestash/logical/examples](docs/guides/mysql/backup/kubestash/logical/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + +## Backup MySQL + +KubeStash supports backups for `MySQL` instances across different configurations, including Standalone, Group Replication, and InnoDB Cluster setups. In this demonstration, we'll focus on a `MySQL` database using Group Replication. The backup and restore process is similar for Standalone and InnoDB Cluster configurations as well. + +This section will demonstrate how to backup a `MySQL` database. Here, we are going to deploy a `MySQL` database using KubeDB. Then, we are going to backup this database into a `GCS` bucket. Finally, we are going to restore the backup up data into another `MySQL` database. + +### Deploy Sample MySQL Database + +Let's deploy a sample `MySQL` database and insert some data into it. + +**Create MySQL CR:** + +Below is the YAML of a sample `MySQL` CRD that we are going to create for this tutorial: + +```yaml +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: sample-mysql + namespace: demo +spec: + version: "8.2.0" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + deletionPolicy: WipeOut +``` + +Here, +- `.spec.topology` specifies about the clustering configuration of MySQL. +- `.Spec.topology.mode` specifies the mode of MySQL Cluster. During the demonstration we consider to use `GroupReplication`. + +Create the above `MySQL` CR, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/sample-mysql.yaml +mysql.kubedb.com/sample-mysql created +``` + +KubeDB will deploy a MySQL database according to the above specification. It will also create the necessary `Secrets` and `Services` to access the database. + +Let's check if the database is ready to use, + +```bash +$ kubectl get mysqls.kubedb.com -n demo +NAME VERSION STATUS AGE +sample-mysql 8.2.0 Ready 4m22s +``` + +The database is `Ready`. Verify that KubeDB has created a `Secret` and a `Service` for this database using the following commands, + +```bash +$ kubectl get secret -n demo -l=app.kubernetes.io/instance=sample-mysql +NAME TYPE DATA AGE +sample-mysql-auth Opaque 2 4m58s + +$ kubectl get service -n demo -l=app.kubernetes.io/instance=sample-mysql +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +sample-mysql ClusterIP 10.96.55.61 3306/TCP 97s +sample-mysql-pods ClusterIP None 3306/TCP 97s +sample-mysql-standby ClusterIP 10.96.211.186 3306/TCP 97 +``` + +Here, we have to use service `sample-mysql` and secret `sample-mysql-auth` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/mysql/concepts/appbinding/index.md) CR that holds the necessary information to connect with the database. + +**Verify AppBinding:** + +Verify that the `AppBinding` has been created successfully using the following command, + +```bash +$ kubectl get appbindings -n demo +NAME AGE +sample-mysql 9m24s +``` + +Let's check the YAML of the above `AppBinding`, + +```bash +$ kubectl get appbindings -n demo sample-mysql -o yaml +``` + +```yaml +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: sample-mysql + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: mysqls.kubedb.com + name: sample-mysql + namespace: demo + ownerReferences: + - apiVersion: kubedb.com/v1 + blockOwnerDeletion: true + controller: true + kind: MySQL + name: sample-mysql + uid: edde3e8b-7775-4f91-85a9-4ba4b96315f7 + resourceVersion: "5126" + uid: 86c9a149-f8ab-44c4-947f-5f9b402aad6c +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo + clientConfig: + service: + name: sample-mysql + path: / + port: 3306 + scheme: tcp + url: tcp(sample-mysql.demo.svc:3306)/ + ... + ... + secret: + name: sample-mysql-auth + type: kubedb.com/mysql + version: 8.2.0 +``` + +KubeStash uses the `AppBinding` CR to connect with the target database. It requires the following two fields to set in AppBinding's `.spec` section. + +- `.spec.clientConfig.service.name` specifies the name of the Service that connects to the database. +- `.spec.secret` specifies the name of the Secret that holds necessary credentials to access the database. +- `spec.type` specifies the types of the app that this AppBinding is pointing to. KubeDB generated AppBinding follows the following format: `/`. + +**Insert Sample Data:** + +Now, we are going to exec into the database pod and create some sample data. At first, find out the database `Pod` using the following command, + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-mysql" +NAME READY STATUS RESTARTS AGE +sample-mysql-0 2/2 Running 0 2m41s +sample-mysql-1 2/2 Running 0 2m35s +sample-mysql-2 2/2 Running 0 2m29s +``` + +And copy the username and password of the `root` user to access into `mysql` shell. + +```bash +$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.username}'| base64 -d +root⏎ + +$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.password}'| base64 -d +DZfmUZd14fNEEOU4⏎ +``` + +Now, Lets exec into the `Pod` to enter into `mysql` shell and create a database and a table, + +```bash +$ kubectl exec -it -n demo sample-mysql-0 -- mysql --user=root --password=DZfmUZd14fNEEOU4 +Defaulted container "mysql" out of: mysql, mysql-init (init) +mysql: [Warning] Using a password on the command line interface can be insecure. +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 977 +Server version: 8.2.0 MySQL Community Server - GPL + +Copyright (c) 2000, 2023, Oracle and/or its affiliates. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> CREATE DATABASE playground; +Query OK, 1 row affected (0.01 sec) + +mysql> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| information_schema | +| mysql | +| performance_schema | +| playground | +| sys | ++--------------------+ +5 rows in set (0.00 sec) + +mysql> CREATE TABLE playground.equipment ( id INT NOT NULL AUTO_INCREMENT, type VARCHAR(50), quant INT, color VARCHAR(25), PRIMARY KEY(id)); +Query OK, 0 rows affected (0.01 sec) + +mysql> SHOW TABLES IN playground; ++----------------------+ +| Tables_in_playground | ++----------------------+ +| equipment | ++----------------------+ +1 row in set (0.01 sec) + +mysql> INSERT INTO playground.equipment (type, quant, color) VALUES ("slide", 2, "blue"); +Query OK, 1 row affected (0.01 sec) + +mysql> SELECT * FROM playground.equipment; ++----+-------+-------+-------+ +| id | type | quant | color | ++----+-------+-------+-------+ +| 1 | slide | 2 | blue | ++----+-------+-------+-------+ +1 row in set (0.00 sec) + +mysql> exit +Bye +``` + +Now, we are ready to backup the database. + +### Prepare Backend + +We are going to store our backed up data into a GCS bucket. We have to create a Secret with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). + +**Create Secret:** + +Let's create a secret called `gcs-secret` with access credentials to our desired GCS bucket, + +```bash +$ echo -n '' > GOOGLE_PROJECT_ID +$ cat /path/to/downloaded-sa-key.json > GOOGLE_SERVICE_ACCOUNT_JSON_KEY +$ kubectl create secret generic -n demo gcs-secret \ + --from-file=./GOOGLE_PROJECT_ID \ + --from-file=./GOOGLE_SERVICE_ACCOUNT_JSON_KEY +secret/gcs-secret created +``` + +**Create BackupStorage:** + +Now, create a `BackupStorage` using this secret. Below is the YAML of `BackupStorage` CR we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete +``` + +Let's create the BackupStorage we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/backupstorage.yaml +backupstorage.storage.kubestash.com/gcs-storage created +``` + +Now, we are ready to backup our database to our desired backend. + +**Create RetentionPolicy:** + +Now, let's create a `RetentionPolicy` to specify how the old Snapshots should be cleaned up. + +Below is the YAML of the `RetentionPolicy` object that we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Let’s create the above `RetentionPolicy`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/retentionpolicy.yaml +retentionpolicy.storage.kubestash.com/demo-retention created +``` + +### Backup + +We have to create a `BackupConfiguration` targeting respective `sample-mysql` MySQL database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. + +At first, we need to create a secret with a Restic password for backup data encryption. + +**Create Secret:** + +Let's create a secret called `encrypt-secret` with the Restic password, + +```bash +$ echo -n 'changeit' > RESTIC_PASSWORD +$ kubectl create secret generic -n demo encrypt-secret \ + --from-file=./RESTIC_PASSWORD \ +secret "encrypt-secret" created +``` + +**Create BackupConfiguration:** + +Below is the YAML for `BackupConfiguration` CR to backup the `sample-mysql` database that we have deployed earlier, + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup +``` + +- `.spec.sessions[*].schedule` specifies that we want to backup the database at `5 minutes` interval. +- `.spec.target` refers to the targeted `sample-mysql` MySQL database that we created earlier. + +Let's create the `BackupConfiguration` CR that we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/backupconfiguration.yaml +backupconfiguration.core.kubestash.com/sample-mysql-backup created +``` + +**Verify Backup Setup Successful** + +If everything goes well, the phase of the `BackupConfiguration` should be `Ready`. The `Ready` phase indicates that the backup setup is successful. Let's verify the `Phase` of the BackupConfiguration, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +sample-mysql-backup Ready 2m50s +``` + +Additionally, we can verify that the `Repository` specified in the `BackupConfiguration` has been created using the following command, + +```bash +$ kubectl get repo -n demo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-mysql-repo 0 0 B Ready 3m +``` + +KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/mysql` directory. + +**Verify CronJob:** + +It will also create a `CronJob` with the schedule specified in `spec.sessions[*].scheduler.schedule` field of `BackupConfiguration` CR. + +Verify that the `CronJob` has been created using the following command, + +```bash +$ kubectl get cronjob -n demo +NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE +trigger-sample-mysql-backup-frequent-backup */5 * * * * 0 2m45s 3m25s +``` + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w + +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +sample-mysql-backup-frequent-backup-1724065200 BackupConfiguration sample-mysql-backup Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-mysql-backup` has been updated by the following command, + +```bash +$ kubectl get repository -n demo gcs-mysql-repo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-mysql-repo true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=gcs-demo-repo +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 sample-mysql-backup frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-08-19T12:10:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubestash.com/app-ref-kind: MySQL + kubestash.com/app-ref-name: sample-mysql + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: gcs-mysql-repo + annotations: + kubedb.com/db-version: 8.2.0 + name: gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: gcs-mysql-repo + uid: 036a2605-0dcc-43bd-a921-363d5c3e8cf0 + resourceVersion: "33742" + uid: d7f14919-4c31-4b46-84e9-0a75f303ad92 +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo + backupSession: sample-mysql-backup-frequent-backup-1724065200 + deletionPolicy: Delete + repository: gcs-mysql-repo + session: frequent-backup + snapshotID: 01J6V4P4J0R33C8EG0JWK82118 + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 7.472499073s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: fbab3af5c38f51b7aa9096799d8ce4b71ea0092dd8297526fed0adbd9f62f3f1 + size: 3.657 MiB + uploaded: 1.036 MiB + size: 1.456 MiB + ... +``` + +> KubeStash uses the `mysqldump` command to take backups of target MySQL databases. Therefore, the component name for logical backups is set as `dump`. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/mysql/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/dep/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Restore + +In this section, we are going to restore the database from the backup we have taken in the previous section. We are going to deploy a new database and initialize it from the backup. + +#### Deploy Restored Database: + +Now, we have to deploy the restored database similarly as we have deployed the original `sample-mysql` database. However, this time there will be the following differences: + +- We are going to specify `.spec.init.waitForInitialRestore` field that tells KubeDB to wait for first restore to complete before marking this database is ready to use. + +Below is the YAML for `MySQL` CRD we are going deploy to initialize from backup, + +```yaml +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: restored-mysql + namespace: demo +spec: + init: + waitForInitialRestore: true + version: "8.2.0" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi + deletionPolicy: WipeOut +``` + +Let's create the above database, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/restored-mysql.yaml +mysql.kubedb.com/restored-mysql created +``` + +If you check the database status, you will see it is stuck in `Provisioning` state. + +```bash +$ kubectl get my -n demo restored-mysql +NAME VERSION STATUS AGE +restored-mysql 8.2.0 Provisioning 61s +``` + +#### Create RestoreSession: + +Now, we need to create a RestoreSession CRD pointing to targeted `MySQL` database. + +Below, is the contents of YAML file of the `RestoreSession` object that we are going to create to restore backed up data into the newly created database provisioned by MySQL object named `restored-mysql`. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: restored-mysql + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore +``` + +Here, + +- `.spec.target` refers to the newly created `restored-mysql` MySQL object to where we want to restore backup data. +- `.spec.dataSource.repository` specifies the Repository object that holds the backed up data. +- `.spec.dataSource.snapshot` specifies to restore from latest `Snapshot`. + +Let's create the RestoreSession CRD object we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/restoresession.yaml +restoresession.core.kubestash.com/sample-mysql-restore created +``` + +Once, you have created the `RestoreSession` object, KubeStash will create restore Job. Run the following command to watch the phase of the `RestoreSession` object, + +```bash +$ watch kubectl get restoresession -n demo +Every 2.0s: kubectl get restores... AppsCode-PC-03: Wed Aug 21 10:44:05 2024 + +NAME REPOSITORY FAILURE-POLICY PHASE DURATION AGE +sample-restore gcs-demo-repo Succeeded 3s 53s +``` + +The `Succeeded` phase means that the restore process has been completed successfully. + + +#### Verify Restored Data: + +In this section, we are going to verify whether the desired data has been restored successfully. We are going to connect to the database server and check whether the database and the table we created earlier in the original database are restored. + +At first, check if the database has gone into `Ready` state by the following command, + +```bash +$ kubectl get my -n demo restored-mysql +NAME VERSION STATUS AGE +restored-mysql 8.2.0 Ready 34m +``` + +Now, find out the database `Pod` by the following command, + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=restored-mysql" +NAME READY STATUS RESTARTS AGE +restored-mysql-0 1/1 Running 0 39m +``` + +And then copy the user name and password of the `root` user to access into `mysql` shell. + +```bash +$ kubectl get secret -n demo restored-mysql-auth -o jsonpath='{.data.username}'| base64 -d +root + +$ kubectl get secret -n demo restored-mysql-auth -o jsonpath='{.data.password}'| base64 -d +QMm1hi0T*7QFz_yh +``` + +Now, let's exec into the Pod to enter into `mysql` shell and verify restored data, + +```bash +$ kubectl exec -it -n demo restored-mysql-0 -- mysql --user=root --password='QMm1hi0T*7QFz_yh' +Defaulted container "mysql" out of: mysql, mysql-coordinator, mysql-init (init) +mysql: [Warning] Using a password on the command line interface can be insecure. +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 243 +Server version: 8.2.0 MySQL Community Server - GPL + +Copyright (c) 2000, 2023, Oracle and/or its affiliates. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| information_schema | +| mysql | +| performance_schema | +| playground | +| sys | ++--------------------+ +5 rows in set (0.00 sec) + +mysql> SHOW TABLES IN playground; ++----------------------+ +| Tables_in_playground | ++----------------------+ +| equipment | ++----------------------+ +1 row in set (0.00 sec) + +mysql> SELECT * FROM playground.equipment; ++----+-------+-------+-------+ +| id | type | quant | color | ++----+-------+-------+-------+ +| 1 | slide | 2 | blue | ++----+-------+-------+-------+ +1 row in set (0.00 sec) + +mysql> exit +Bye +``` + +So, from the above output, we can see that the `playground` database and the `equipment` table we have created earlier in the original database and now, they are restored successfully. + +## Cleanup + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl delete backupconfigurations.core.kubestash.com -n demo sample-mysql-backup +kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-mysql +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention +kubectl delete backupstorage -n demo gcs-storage +kubectl delete secret -n demo gcs-secret +kubectl delete secret -n demo encrypt-secret +kubectl delete my -n demo restored-mysql +kubectl delete my -n demo sample-mysql +``` \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/overview/images/backup_overview.svg b/docs/guides/singlestore/backup/kubestash/overview/images/backup_overview.svg new file mode 100644 index 0000000000..9d245e0dcd --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/overview/images/backup_overview.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/overview/images/kubedb_plus_kubestash.svg b/docs/guides/singlestore/backup/kubestash/overview/images/kubedb_plus_kubestash.svg new file mode 100644 index 0000000000..380d92d969 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/overview/images/kubedb_plus_kubestash.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/guides/singlestore/backup/kubestash/overview/images/restore_overview.svg b/docs/guides/singlestore/backup/kubestash/overview/images/restore_overview.svg new file mode 100644 index 0000000000..f2c2488962 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/overview/images/restore_overview.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/overview/index.md b/docs/guides/singlestore/backup/kubestash/overview/index.md new file mode 100644 index 0000000000..1853aedbe0 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/overview/index.md @@ -0,0 +1,98 @@ +--- +title: Backup & Restore SingleStore Using KubeStash +menu: + docs_{{ .version }}: + identifier: guides-sdb-backup-overview-stashv2 + name: Overview + parent: guides-sdb-backup-stashv2 + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +{{< notice type="warning" message="Please install [KubeStash](https://kubestash.com/docs/latest/setup/install/kubestash/) to try this feature. Database backup with KubeStash is already included in the KubeDB license. So, you don't need a separate license for KubeStash." >}} + +# SingleStore Backup & Restore Overview + +KubeDB also uses [KubeStash](https://kubestash.com) to backup and restore databases. KubeStash by AppsCode is a cloud native data backup and recovery solution for Kubernetes workloads and databases. KubeStash utilizes [restic](https://github.com/restic/restic) to securely backup stateful applications to any cloud or on-prem storage backends (for example, S3, GCS, Azure Blob storage, Minio, NetApp, Dell EMC etc.). + +
+  KubeDB + KubeStash +
Fig: Backup KubeDB Databases Using KubeStash
+
+ +## How Backup Works + +The following diagram shows how KubeStash takes backup of a `SingleStore` database. Open the image in a new tab to see the enlarged version. + +
+  SingleStore Backup Overview +
Fig: SingleStore Backup Overview
+
+ +The backup process consists of the following steps: + +1. At first, a user creates a `Secret`. This secret holds the credentials to access the backend where the backed up data will be stored. + +2. Then, she creates a `BackupStorage` custom resource that specifies the backend information, along with the `Secret` containing the credentials needed to access the backend. + +3. KubeStash operator watches for `BackupStorage` custom resources. When it finds a `BackupStorage` object, it initializes the `BackupStorage` by uploading the `metadata.yaml` file to the specified backend. + +4. Next, she creates a `BackupConfiguration` custom resource that specifies the target database, addon information (including backup tasks), backup schedules, storage backends for storing the backup data, and other additional settings. + +5. KubeStash operator watches for `BackupConfiguration` objects. + +6. Once the KubeStash operator finds a `BackupConfiguration` object, it creates `Repository` with the information specified in the `BackupConfiguration`. + +7. KubeStash operator watches for `Repository` custom resources. When it finds the `Repository` object, it Initializes `Repository` by uploading `repository.yaml` file into the `spec.sessions[*].repositories[*].directory` path specified in `BackupConfiguration`. + +8. Then, it creates a `CronJob` for each session with the schedule specified in `BackupConfiguration` to trigger backup periodically. + +9. KubeStash operator triggers an instant backup as soon as the `BackupConfiguration` is ready. Backups are otherwise triggered by the `CronJob` based on the specified schedule. + +10. KubeStash operator watches for `BackupSession` custom resources. + +11. When it finds a `BackupSession` object, it creates a `Snapshot` custom resource for each `Repository` specified in the `BackupConfiguration`. + +12. Then it resolves the respective `Addon` and `Function` and prepares backup `Job` definition. + +13. Then, it creates the `Job` to backup the targeted `SingleStore` database. + +14. The backup `Job` reads necessary information (e.g. auth secret, port) to connect with the database from the `AppBinding` CR. It also reads backend information and access credentials from `BackupStorage` CR, Storage Secret and `Repository` path respectively. + +15. Then, the `Job` dumps the targeted `SingleStore` database and uploads the output to the backend. KubeStash pipes the output of dump command to uploading process. Hence, backup `Job` does not require a large volume to hold the entire dump output. + +16. After the backup process is completed, the backup `Job` updates the `status.components[dump]` field of the `Snapshot` resources with backup information of the target `SingleStore` database. + +## How Restore Process Works + +The following diagram shows how KubeStash restores backed up data into a `SingleStore` database. Open the image in a new tab to see the enlarged version. + +
+  Database Restore Overview +
Fig: SingleStore Restore Process Overview
+
+ +The restore process consists of the following steps: + +1. At first, a user creates a `SingleStore` database where the data will be restored or the user can use the same `SingleStore` database. + +2. Then, she creates a `RestoreSession` custom resource that specifies the target database where the backed-up data will be restored, addon information (including restore tasks), the target snapshot to be restored, the Repository containing that snapshot, and other additional settings. + +3. KubeStash operator watches for `RestoreSession` custom resources. + +4. When it finds a `RestoreSession` custom resource, it resolves the respective `Addon` and `Function` and prepares a restore `Job` definition. + +5. Then, it creates the `Job` to restore the target. + +6. The `Job` reads necessary information to connect with the database from respective `AppBinding` CR. It also reads backend information and access credentials from `Repository` CR and storage `Secret` respectively. + +7. Then, the `Job` downloads the backed up data from the backend and injects into the desired database. KubeStash pipes the downloaded data to the respective database tool to inject into the database. Hence, restore `Job` does not require a large volume to download entire backup data inside it. + +8. Finally, when the restore process is completed, the `Job` updates the `status.components[*]` field of the `RestoreSession` with restore information of the target database. + +## Next Steps + +- Backup a `SingleStore` database using KubeStash by the following guides from [here](/docs/guides/singlestore/backup/kubestash/logical/index.md). \ No newline at end of file From 63eddde3d0526dbb8bfbb9b1823cc19b6b0bfdba Mon Sep 17 00:00:00 2001 From: Ashraful Haque Tani Date: Thu, 19 Sep 2024 16:44:51 +0600 Subject: [PATCH 06/10] add customize, autobackup Signed-off-by: Ashraful Haque Tani --- .../auto-backup/examples/backupstorage.yaml | 17 + .../examples/customize-backupblueprint.yaml | 41 + .../examples/default-backupblueprint.yaml | 37 + .../auto-backup/examples/retentionpolicy.yaml | 15 + .../examples/sample-singlestore-2.yaml | 57 ++ .../examples/sample-singlestore.yaml | 53 ++ .../backup/kubestash/auto-backup/index.md | 812 ++++++++++++++++++ .../examples/backup/multiple-backends.yaml | 49 ++ .../examples/backup/passing-args.yaml | 38 + .../examples/backup/passing-databases.yaml | 37 + .../examples/backup/resources-limit.yaml | 46 + .../examples/backup/specific-user.yaml | 41 + .../examples/common/gcs-backupstorage.yaml | 17 + .../examples/common/retentionpolicy.yaml | 15 + .../examples/common/s3-backupstorage.yaml | 19 + .../examples/common/sample-singlestore.yaml | 50 ++ .../examples/restore/passing-args.yaml | 23 + .../examples/restore/resources-limit.yaml | 30 + .../examples/restore/specific-snapshot.yaml | 21 + .../examples/restore/specific-user.yaml | 26 + .../backup/kubestash/customization/index.md | 427 +++++++++ 21 files changed, 1871 insertions(+) create mode 100644 docs/guides/singlestore/backup/kubestash/auto-backup/examples/backupstorage.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/auto-backup/examples/retentionpolicy.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore-2.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/auto-backup/index.md create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/backup/multiple-backends.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/backup/passing-args.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/backup/passing-databases.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/backup/resources-limit.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/backup/specific-user.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/common/gcs-backupstorage.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/common/retentionpolicy.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/common/s3-backupstorage.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/common/sample-singlestore.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/restore/passing-args.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/restore/resources-limit.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/restore/specific-snapshot.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/examples/restore/specific-user.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/customization/index.md diff --git a/docs/guides/singlestore/backup/kubestash/auto-backup/examples/backupstorage.yaml b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/backupstorage.yaml new file mode 100644 index 0000000000..f5478706b6 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: blueprint + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml new file mode 100644 index 0000000000..83509dc609 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml @@ -0,0 +1,41 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: singlestore-customize-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + # ============== Blueprint for Backends of BackupConfiguration ================= + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + # ============== Blueprint for Sessions of BackupConfiguration ================= + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: ${schedule} + jobTemplate: + backoffLimit: 1 + repositories: + - name: ${repoName} + backend: gcs-backend + directory: ${namespace}/${targetName} + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup + params: + databases: ${targetedDatabases} \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml new file mode 100644 index 0000000000..ed86b82a42 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml @@ -0,0 +1,37 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: singlestore-default-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: default-blueprint + backend: gcs-backend + directory: /default-blueprint + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/auto-backup/examples/retentionpolicy.yaml b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore-2.yaml b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore-2.yaml new file mode 100644 index 0000000000..d951719db8 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore-2.yaml @@ -0,0 +1,57 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: sample-singlestore-2 + namespace: demo + annotations: + blueprint.kubestash.com/name: singlestore-customize-backup-blueprint + blueprint.kubestash.com/namespace: demo + variables.kubestash.com/schedule: "*/10 * * * *" + variables.kubestash.com/repoName: customize-blueprint + variables.kubestash.com/namespace: demo + variables.kubestash.com/targetName: sample-singlestore-2 +spec: + version: "8.7.10" + topology: + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + storageType: Durable + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore.yaml b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore.yaml new file mode 100644 index 0000000000..3cafc0950e --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore.yaml @@ -0,0 +1,53 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: sample-singlestore + namespace: demo + annotations: + blueprint.kubestash.com/name: singlestore-default-backup-blueprint + blueprint.kubestash.com/namespace: demo +spec: + version: "8.7.10" + topology: + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + storageType: Durable + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/auto-backup/index.md b/docs/guides/singlestore/backup/kubestash/auto-backup/index.md new file mode 100644 index 0000000000..0fb1a47680 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/auto-backup/index.md @@ -0,0 +1,812 @@ +--- +title: SingleStore Auto-Backup | KubeStash +description: Backup SingleStore using KubeStash Auto-Backup +menu: + docs_{{ .version }}: + identifier: guides-sdb-backup-auto-backup-stashv2 + name: Auto-Backup + parent: guides-sdb-backup-stashv2 + weight: 30 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Backup SingleStore using KubeStash Auto-Backup + +KubeStash can automatically be configured to backup any `SingleStore` databases in your cluster. KubeStash enables cluster administrators to deploy backup `blueprints` ahead of time so database owners can easily backup any `SingleStore` database with a few annotations. + +In this tutorial, we are going to show how you can configure a backup blueprint for `SingleStore` databases in your cluster and backup them with a few annotations. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. +- Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). +- Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). +- Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- If you are not familiar with how KubeStash backup and restore SingleStore databases, please check the following guide [here](/docs/guides/singlestore/backup/kubestash/overview/index.md). + +You should be familiar with the following `KubeStash` concepts: + +- [BackupStorage](https://kubestash.com/docs/latest/concepts/crds/backupstorage/) +- [BackupConfiguration](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/) +- [BackupSession](https://kubestash.com/docs/latest/concepts/crds/backupsession/) +- [RestoreSession](https://kubestash.com/docs/latest/concepts/crds/restoresession/) +- [Addon](https://kubestash.com/docs/latest/concepts/crds/addon/) +- [Function](https://kubestash.com/docs/latest/concepts/crds/function/) +- [Task](https://kubestash.com/docs/latest/concepts/crds/addon/#task-specification) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +### Prepare Backend + +We are going to store our backed up data into a GCS bucket. We have to create a Secret with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). + +**Create Secret:** + +Let's create a secret called `gcs-secret` with access credentials to our desired GCS bucket, + +```bash +$ echo -n '' > GOOGLE_PROJECT_ID +$ cat /path/to/downloaded-sa-key.json > GOOGLE_SERVICE_ACCOUNT_JSON_KEY +$ kubectl create secret generic -n demo gcs-secret \ + --from-file=./GOOGLE_PROJECT_ID \ + --from-file=./GOOGLE_SERVICE_ACCOUNT_JSON_KEY +secret/gcs-secret created +``` + +**Create BackupStorage:** + +Now, create a `BackupStorage` using this secret. Below is the YAML of `BackupStorage` CR we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: blueprint + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete +``` + +Let's create the BackupStorage we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/auto-backup/examples/backupstorage.yaml +backupstorage.storage.kubestash.com/gcs-storage created +``` + +**Create RetentionPolicy:** + +Now, let's create a `RetentionPolicy` to specify how the old Snapshots should be cleaned up. + +Below is the YAML of the `RetentionPolicy` object that we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Let’s create the above `RetentionPolicy`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/auto-backup/examples/retentionpolicy.yaml +retentionpolicy.storage.kubestash.com/demo-retention created +``` + +**Create Secret:** + +We also need to create a secret with a `Restic` password for backup data encryption. + +Let's create a secret called `encrypt-secret` with the Restic password, + +```bash +$ echo -n 'changeit' > RESTIC_PASSWORD +$ kubectl create secret generic -n demo encrypt-secret \ + --from-file=./RESTIC_PASSWORD +secret "encrypt-secret" created +``` + +## Auto-backup with default configurations + +In this section, we are going to backup a `SingleStore` database of `demo` namespace. We are going to use the default configurations which will be specified in the `Backup Blueprint` CR. + +**Prepare Backup Blueprint** + +A `BackupBlueprint` allows you to specify a template for the `Repository`,`Session` or `Variables` of `BackupConfiguration` in a Kubernetes native way. + +Now, we have to create a `BackupBlueprint` CR with a blueprint for `BackupConfiguration` object. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: singlestore-default-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: default-blueprint + backend: gcs-backend + directory: /default-blueprint + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup +``` + +Here, + +- `.spec.backupConfigurationTemplate.backends[*].storageRef` refers our earlier created `gcs-storage` backupStorage. +- `.spec.backupConfigurationTemplate.sessions[*].schedule` specifies that we want to backup the database at `5 minutes` interval. + +Let's create the `BackupBlueprint` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/auto-backup/examples/default-backupblueprint.yaml +backupblueprint.core.kubestash.com/singlestore-default-backup-blueprint created +``` + +Now, we are ready to backup our `SingleStore` databases using few annotations. + +**Create SingleStore License Secret** + +We need SingleStore License to create SingleStore Database. So, Ensure that you have acquired a license and then simply pass the license by secret. + +```bash +$ kubectl create secret generic -n demo license-secret \ + --from-literal=username=license \ + --from-literal=password='your-license-set-here' +secret/license-secret created +``` + +**Create Database** + +Now, we are going to create an `SingleStore` CR in demo namespace. Below is the YAML of the SingleStore object that we are going to create, + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: sample-singlestore + namespace: demo + annotations: + blueprint.kubestash.com/name: singlestore-default-backup-blueprint + blueprint.kubestash.com/namespace: demo +spec: + version: "8.7.10" + topology: + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + storageType: Durable + deletionPolicy: WipeOut +``` + +Here, + +- `.spec.annotations.blueprint.kubestash.com/name: singlestore-default-backup-blueprint` specifies the name of the `BackupBlueprint` that will use in backup. +- `.spec.annotations.blueprint.kubestash.com/namespace: demo` specifies the name of the `namespace` where the `BackupBlueprint` resides. + +Let's create the `SingleStore` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore.yaml +singlestore.kubedb.com/sample-singlestore created +``` + +**Verify BackupConfiguration** + +If everything is set up correctly, KubeStash will create a `BackupConfiguration` for our SingleStore instance in the demo namespace. The phase of this `BackupConfiguration` should be Ready. You can verify the `BackupConfiguration` object by running the following command, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +appbinding-sample-singlestore Ready 2m50m +``` + +Now, let’s check the YAML of the `BackupConfiguration`. + +```bash +$ kubectl get backupconfiguration -n demo appbinding-sample-singlestore -o yaml +``` + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + creationTimestamp: "2024-08-21T09:47:34Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + app.kubernetes.io/managed-by: kubestash.com + kubestash.com/invoker-name: singlestore-default-backup-blueprint + kubestash.com/invoker-namespace: demo + name: appbinding-sample-singlestore + namespace: demo + resourceVersion: "113911" + uid: eef4c853-4df6-4b5e-b462-977c9b2188c0 +spec: + backends: + - name: gcs-backend + retentionPolicy: + name: demo-retention + namespace: demo + storageRef: + name: gcs-storage + namespace: demo + sessions: + - addon: + name: singlestore-addon + tasks: + - name: logical-backup + name: frequent-backup + repositories: + - backend: gcs-backend + directory: /default-blueprint + encryptionSecret: + name: encrypt-secret + namespace: demo + name: default-blueprint + scheduler: + jobTemplate: + backoffLimit: 1 + template: + controller: {} + metadata: {} + spec: + resources: {} + schedule: '*/5 * * * *' + sessionHistoryLimit: 3 + target: + apiGroup: kubedb.com + kind: Singlestore + name: sample-singlestore + namespace: demo +``` + +Notice the `spec.backends`, `spec.sessions` and `spec.target` sections, KubeStash automatically resolved those info from the `BackupBluePrint` and created above `BackupConfiguration`. + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w + +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +appbinding-sample-singlestore-frequent-backup-1724236500 BackupConfiguration appbinding-sample-singlestore Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `default-blueprint` has been updated by the following command, + +```bash +$ kubectl get repository -n demo default-blueprint +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +default-blueprint true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=default-blueprint +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +default-blueprint-appbinding-sample-singlestore-frequent-backup-1724236500 default-blueprint frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubedb.com/db-version: ` +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo default-blueprint-appbinding-sample-singlestore-frequent-backup-1724236500 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-08-21T10:35:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubedb.com/db-version: 8.7.10 + kubestash.com/app-ref-kind: Singlestore + kubestash.com/app-ref-name: sample-singlestore + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: default-blueprint + name: default-blueprint-appbinding-sample-singlestore-frequent-backup-1724236500 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: default-blueprint + uid: 61e771fc-8262-480c-a9e7-3c5c11c8fd77 + resourceVersion: "118423" + uid: 27e2235a-22c1-449a-be92-c53506fe1fe4 +spec: + appRef: + apiGroup: kubedb.com + kind: Singlestore + name: sample-singlestore + namespace: demo + backupSession: appbinding-sample-singlestore-frequent-backup-1724236500 + deletionPolicy: Delete + repository: default-blueprint + session: frequent-backup + snapshotID: 01J6V48XS6QM489WPKX1MDD4W9 + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 6.692337543s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: b83d7a5577940d1c8f5bcda0630592c7d5a04168c272c0e7560bf7dacfe35ea8 + size: 3.657 MiB + uploaded: 121.343 KiB + size: 772.958 KiB + integrity: true + phase: Succeeded + size: 772.957 KiB + snapshotTime: "2024-08-21T10:35:00Z" + totalComponents: 1 +``` + +> KubeStash uses the `mysqldump` command to take backups of target SingleStore databases. Therefore, the component name for `logical backups` is set as `dump`. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `/blueprint/default-blueprint/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/default-blueprintrepository/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Auto-backup with custom configurations + +In this section, we are going to backup a `SingleStore` database of `demo` namespace. We are going to use the custom configurations which will be specified in the `BackupBlueprint` CR. + +**Prepare Backup Blueprint** + +A `BackupBlueprint` allows you to specify a template for the `Repository`,`Session` or `Variables` of `BackupConfiguration` in a Kubernetes native way. + +Now, we have to create a `BackupBlueprint` CR with a blueprint for `BackupConfiguration` object. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupBlueprint +metadata: + name: singlestore-customize-backup-blueprint + namespace: demo +spec: + usagePolicy: + allowedNamespaces: + from: All + backupConfigurationTemplate: + deletionPolicy: OnDelete + # ============== Blueprint for Backends of BackupConfiguration ================= + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + # ============== Blueprint for Sessions of BackupConfiguration ================= + sessions: + - name: frequent-backup + sessionHistoryLimit: 3 + scheduler: + schedule: ${schedule} + jobTemplate: + backoffLimit: 1 + repositories: + - name: ${repoName} + backend: gcs-backend + directory: ${namespace}/${targetName} + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup + params: + databases: ${targetedDatabases} +``` + +Note that we have used some variables (format: `${}`) in different fields. KubeStash will substitute these variables with values from the respective target’s annotations. You’re free to use any variables you like. + +Here, + +- `.spec.backupConfigurationTemplate.backends[*].storageRef` refers our earlier created `gcs-storage` backupStorage. +- `.spec.backupConfigurationTemplate.sessions[*]`: + - `.schedule` defines `${schedule}` variable, which determines the time interval for the backup. + - `.repositories[*].name` defines the `${repoName}` variable, which specifies the name of the backup `Repository`. + - `.repositories[*].directory` defines two variables, `${namespace}` and `${targetName}`, which are used to determine the path where the backup will be stored. + - `.addon.tasks[*]databases` defines `${targetedDatabases}` variable, which identifies list of databases to backup. + +Let's create the `BackupBlueprint` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/auto-backup/examples/customize-backupblueprint.yaml +backupblueprint.core.kubestash.com/singlestore-customize-backup-blueprint created +``` + +Now, we are ready to backup our `SingleStore` databases using few annotations. You can check available auto-backup annotations for a databases from [here](https://kubestash.com/docs/latest/concepts/crds/backupblueprint/). + +**Create Database** + +Now, we are going to create an `SingleStore` CR in demo namespace. Below is the YAML of the SingleStore object that we are going to create, + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: sample-singlestore-2 + namespace: demo + annotations: + blueprint.kubestash.com/name: singlestore-customize-backup-blueprint + blueprint.kubestash.com/namespace: demo + variables.kubestash.com/schedule: "*/10 * * * *" + variables.kubestash.com/repoName: customize-blueprint + variables.kubestash.com/namespace: demo + variables.kubestash.com/targetName: sample-singlestore-2 +spec: + version: "8.7.10" + topology: + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + storageType: Durable + deletionPolicy: WipeOut +``` + +Notice the `metadata.annotations` field, where we have defined the annotations related to the automatic backup configuration. Specifically, we've set the `BackupBlueprint` name as `singlestore-customize-backup-blueprint` and the namespace as `demo`. We have also provided values for the blueprint template variables, such as the backup `schedule`, `repositoryName`, `namespace`, and `targetName`. These annotations will be used to create a `BackupConfiguration` for this `SingleStore` database. + +Let's create the `SingleStore` we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/auto-backup/examples/sample-singlestore-2.yaml +singlestore.kubedb.com/sample-singlestore-2 created +``` + +**Verify BackupConfiguration** + +If everything goes well, KubeStash should create a `BackupConfiguration` for our SingleStore in demo namespace and the phase of that `BackupConfiguration` should be `Ready`. Verify the `BackupConfiguration` object by the following command, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +appbinding-sample-singlestore-2 Ready 2m50m +``` + +Now, let’s check the YAML of the `BackupConfiguration`. + +```bash +$ kubectl get backupconfiguration -n demo appbinding-sample-singlestore-2 -o yaml +``` + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + creationTimestamp: "2024-08-21T12:55:38Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + app.kubernetes.io/managed-by: kubestash.com + kubestash.com/invoker-name: singlestore-customize-backup-blueprint + kubestash.com/invoker-namespace: demo + name: appbinding-sample-singlestore-2 + namespace: demo + resourceVersion: "129124" + uid: eb42b736-c9c9-4280-8379-bbb581790185 +spec: + backends: + - name: gcs-backend + retentionPolicy: + name: demo-retention + namespace: demo + storageRef: + name: gcs-storage + namespace: demo + sessions: + - addon: + name: singlestore-addon + tasks: + - name: logical-backup + name: frequent-backup + repositories: + - backend: gcs-backend + directory: demo/sample-singlestore-2 + encryptionSecret: + name: encrypt-secret + namespace: demo + name: customize-blueprint + scheduler: + jobTemplate: + backoffLimit: 1 + template: + controller: {} + metadata: {} + spec: + resources: {} + schedule: '*/10 * * * *' + sessionHistoryLimit: 3 + target: + apiGroup: kubedb.com + kind: Singlestore + name: sample-singlestore-2 + namespace: demo +``` + +Notice the `spec.backends`, `spec.sessions` and `spec.target` sections, KubeStash automatically resolved those info from the `BackupBluePrint` and created above `BackupConfiguration`. + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +```bash +$ kubectl get backupsession -n demo -w + +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +appbinding-sample-singlestore-2-frequent-backup-1725007200 BackupConfiguration appbinding-sample-singlestore-2 Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `customize-blueprint` has been updated by the following command, + +```bash +$ kubectl get repository -n demo customize-blueprint +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +customize-blueprint true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=customize-blueprint +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +customize-blueprint-appbinding-sample-singlestore-2-frequent-backup-1725007200 customize-blueprint frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubedb.com/db-version: ` +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo customize-blueprint-appbinding-sample-singlestore-2-frequent-backup-1725007200 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-08-21T10:35:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubedb.com/db-version: 8.7.10 + kubestash.com/app-ref-kind: Singlestore + kubestash.com/app-ref-name: sample-singlestore + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: customize-blueprint + name: customize-blueprint-appbinding-sample-singlestore-2-frequent-backup-1725007200 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: customize-blueprint + uid: 61e771fc-8262-480c-a9e7-3c5c11c8fd77 + resourceVersion: "118423" + uid: 27e2235a-22c1-449a-be92-c53506fe1fe4 +spec: + appRef: + apiGroup: kubedb.com + kind: Singlestore + name: sample-singlestore-2 + namespace: demo + backupSession: appbinding-sample-singlestore-2-frequent-backup-1725007200 + deletionPolicy: Delete + repository: customize-blueprint + session: frequent-backup + snapshotID: 01J6V48XZW2BJ02GSS4YBW3TWX + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 6.692337543s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: b83d7a5577940d1c8f5bcda0630592c7d5a04168c272c0e7560bf7dacfe35ea8 + size: 3.657 MiB + uploaded: 121.343 KiB + size: 772.958 KiB + integrity: true + phase: Succeeded + size: 772.957 KiB + snapshotTime: "2024-08-21T10:35:00Z" + totalComponents: 1 +``` + +> KubeStash uses the `mysqldump` command to take backups of target SingleStore databases. Therefore, the component name for `logical backups` is set as `dump`. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `/blueprint/custom-blueprint/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `blueprint/custom-blueprint/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Cleanup + +To cleanup the resources crated by this tutorial, run the following commands, + +```bash +kubectl delete backupblueprints.core.kubestash.com -n demo singlestore-default-backup-blueprint +kubectl delete backupblueprints.core.kubestash.com -n demo singlestore-customize-backup-blueprint +kubectl delete backupstorage -n demo gcs-storage +kubectl delete secret -n demo gcs-secret +kubectl delete secret -n demo encrypt-secret +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention +kubectl delete my -n demo sample-singlestore +kubectl delete my -n demo sample-singlestore-2 +``` \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/backup/multiple-backends.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/multiple-backends.yaml new file mode 100644 index 0000000000..7165e33400 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/multiple-backends.yaml @@ -0,0 +1,49 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + - name: s3-singlestore-repo + backend: s3-backend + directory: /singlestore-copy + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/backup/passing-args.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/passing-args.yaml new file mode 100644 index 0000000000..d297d866e0 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/passing-args.yaml @@ -0,0 +1,38 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup + params: + args: --set-gtid-purged=OFF \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/backup/passing-databases.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/passing-databases.yaml new file mode 100644 index 0000000000..cfff9b2b77 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/passing-databases.yaml @@ -0,0 +1,37 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup + params: + databases: db1,db2,db3 \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/backup/resources-limit.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/resources-limit.yaml new file mode 100644 index 0000000000..a7d24412e4 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/resources-limit.yaml @@ -0,0 +1,46 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup + diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/backup/specific-user.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/specific-user.yaml new file mode 100644 index 0000000000..62a56355b7 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/backup/specific-user.yaml @@ -0,0 +1,41 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/common/gcs-backupstorage.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/common/gcs-backupstorage.yaml new file mode 100644 index 0000000000..6ab3df02ac --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/common/gcs-backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/common/retentionpolicy.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/common/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/common/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/common/s3-backupstorage.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/common/s3-backupstorage.yaml new file mode 100644 index 0000000000..a0f1c3ade3 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/common/s3-backupstorage.yaml @@ -0,0 +1,19 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: s3-storage + namespace: demo +spec: + storage: + provider: s3 + s3: + bucket: kubestash + region: us-east-1 + endpoint: us-east-1.linodeobjects.com + secretName: s3-secret + prefix: demo + usagePolicy: + allowedNamespaces: + from: All + default: false + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/common/sample-singlestore.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/common/sample-singlestore.yaml new file mode 100644 index 0000000000..f5123df643 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/common/sample-singlestore.yaml @@ -0,0 +1,50 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: sdb-sample + namespace: demo +spec: + version: "8.7.10" + topology: + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + storageType: Durable + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/restore/passing-args.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/restore/passing-args.yaml new file mode 100644 index 0000000000..fd37ffb7d4 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/restore/passing-args.yaml @@ -0,0 +1,23 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-singlestore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: restored-singlestore + dataSource: + repository: gcs-singlestore-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup-restore + params: + args: --one-database=testdb \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/restore/resources-limit.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/restore/resources-limit.yaml new file mode 100644 index 0000000000..c4d79c7c53 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/restore/resources-limit.yaml @@ -0,0 +1,30 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-singlestore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: restored-singlestore + dataSource: + repository: gcs-singlestore-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup-restore \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/restore/specific-snapshot.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/restore/specific-snapshot.yaml new file mode 100644 index 0000000000..8350581145 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/restore/specific-snapshot.yaml @@ -0,0 +1,21 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-singlestore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: restored-singlestore + dataSource: + repository: gcs-singlestore-repo + snapshot: gcs-singlestore-repo-sample-singlestore-backup-frequent-backup-1725257849 + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup-restore diff --git a/docs/guides/singlestore/backup/kubestash/customization/examples/restore/specific-user.yaml b/docs/guides/singlestore/backup/kubestash/customization/examples/restore/specific-user.yaml new file mode 100644 index 0000000000..98f9c51ffe --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/examples/restore/specific-user.yaml @@ -0,0 +1,26 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-singlestore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: restored-singlestore + dataSource: + repository: gcs-singlestore-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup-restore diff --git a/docs/guides/singlestore/backup/kubestash/customization/index.md b/docs/guides/singlestore/backup/kubestash/customization/index.md new file mode 100644 index 0000000000..c276cef037 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/customization/index.md @@ -0,0 +1,427 @@ +--- +title: SingleStore Backup Customization | KubeStash +description: Customizing SingleStore Backup and Restore process with KubeStash +menu: + docs_{{ .version }}: + identifier: guides-sdb-backup-customization-stashv2 + name: Customizing Backup & Restore Process + parent: guides-sdb-backup-stashv2 + weight: 50 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Customizing Backup and Restore Process + +KubeStash provides rich customization supports for the backup and restore process to meet the requirements of various cluster configurations. This guide will show you some examples of these customizations. + +## Customizing Backup Process + +In this section, we are going to show you how to customize the backup process. Here, we are going to show some examples of providing arguments to the backup process, running the backup process as a specific user, etc. + +### Passing targeted databases to the backup process + +KubeStash SingleStore addon uses the [mysqldump](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html) for backup. Addon has implemented a `databases` params which indicates your targeted backup databases. + +The below example shows how you can pass the `--databases` option during backup. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup + params: + databases: db1,db2,db3 +``` + +> **WARNING**: Make sure that your provides databases has been created before taking backup. + +Here, +- `addon.tasks[*].databases` options indicates targeted databases. By default `singlestore-addon` add `--all-databases` options during backup. If you want to backup all databases keep the `databases` params empty. + +### Passing arguments to the backup process + +KubeStash SingleStore addon uses [mysqldump](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html) for backup. You can pass arguments to the `mysqldump` supported options through `args` param under `addon.tasks[*].params` section. + +The below example shows how you can pass the `--set-gtid-purged=OFF` options during backup. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup + params: + args: --set-gtid-purged=OFF +``` + +### Using multiple backends + +You can configure multiple backends within a single `backupConfiguration`. To back up the same data to different backends, such as S3 and GCS, declare each backend in the `.spe.backends` section. Then, reference these backends in the `.spec.sessions[*].repositories` section. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + - name: s3-singlestore-repo + backend: s3-backend + directory: /singlestore-copy + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup +``` + +### Running backup job as a specific user + +If your cluster requires running the backup job as a specific user, you can provide `securityContext` under `addon.jobTemplate.spec.securityContext` section. The below example shows how you can run the backup job as the `root` user. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup +``` + +### Specifying Memory/CPU limit/request for the backup job + +If you want to specify the Memory/CPU limit/request for your backup job, you can specify `resources` field under `addon.jobTemplate.spec` section. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-singlestore-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: sample-singlestore + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-singlestore-repo + backend: gcs-backend + directory: /singlestore + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup +``` + +> You can configure additional runtime settings for backup jobs within the `addon.jobTemplate.spec` sections. For further details, please refer to the [reference](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/#podtemplate-spec). + +## Customizing Restore Process + +KubeStash also uses `singlestore` during the restore process. In this section, we are going to show how you can pass arguments to the restore process, restore a specific snapshot, run restore job as a specific user, etc. + + +### Passing arguments to the restore process + +Similar to the backup process, you can pass arguments to the restore process through the `args` params under `addon.tasks[*].params` section. + +This example will restore data from database `testdb` only. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-singlestore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: restored-singlestore + dataSource: + repository: gcs-singlestore-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup-restore + params: + args: --one-database=testdb +``` + +### Restore specific snapshot + +You can also restore a specific snapshot. At first, list the available snapshot as bellow, + +```bash +➤ kubectl get snapshots.storage.kubestash.com -n demo -l=kubestash.com/repo-name=gcs-singlestore-repo +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +gcs-singlestore-repo-sample-singlestore-backup-frequent-backup-1725257849 gcs-singlestore-repo frequent-backup 2024-09-02T06:18:01Z Delete Succeeded 15m +gcs-singlestore-repo-sample-singlestore-backup-frequent-backup-1725258000 gcs-singlestore-repo frequent-backup 2024-09-02T06:20:00Z Delete Succeeded 13m +gcs-singlestore-repo-sample-singlestore-backup-frequent-backup-1725258300 gcs-singlestore-repo frequent-backup 2024-09-02T06:25:00Z Delete Succeeded 8m34s +gcs-singlestore-repo-sample-singlestore-backup-frequent-backup-1725258600 gcs-singlestore-repo frequent-backup 2024-09-02T06:30:00Z Delete Succeeded 3m34s +``` + +The below example shows how you can pass a specific snapshot name in `.dataSource` section. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-singlestore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: restored-singlestore + dataSource: + repository: gcs-singlestore-repo + snapshot: gcs-singlestore-repo-sample-singlestore-backup-frequent-backup-1725257849 + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + tasks: + - name: logical-backup-restore + +``` + + +### Running restore job as a specific user + +Similar to the backup process under the `addon.jobTemplate.spec.` you can provide `securityContext` to run the restore job as a specific user. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-singlestore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: restored-singlestore + dataSource: + repository: gcs-singlestore-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + jobTemplate: + spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + tasks: + - name: logical-backup-restore + +``` + +### Specifying Memory/CPU limit/request for the restore job + +Similar to the backup process, you can also provide `resources` field under the `addon.jobTemplate.spec.resources` section to limit the Memory/CPU for your restore job. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-singlestore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Singlestore + namespace: demo + name: restored-singlestore + dataSource: + repository: gcs-singlestore-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: singlestore-addon + jobTemplate: + spec: + resources: + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "200m" + memory: "1Gi" + tasks: + - name: logical-backup-restore +``` + +> You can configure additional runtime settings for restore jobs within the `addon.jobTemplate.spec` sections. For further details, please refer to the [reference](https://kubestash.com/docs/latest/concepts/crds/restoresession/#podtemplate-spec). \ No newline at end of file From c02250f9431d075c6ce0c80f2e940a108d89d877 Mon Sep 17 00:00:00 2001 From: Ashraful Haque Tani Date: Thu, 19 Sep 2024 18:11:58 +0600 Subject: [PATCH 07/10] add application-level Signed-off-by: Ashraful Haque Tani --- .../examples/backupconfiguration.yaml | 37 + .../examples/backupstorage.yaml | 17 + .../examples/restoresession.yaml | 21 + .../examples/retentionpolicy.yaml | 15 + .../examples/sample-singlestore.yaml | 50 ++ .../kubestash/application-level/index.md | 804 ++++++++++++++++++ 6 files changed, 944 insertions(+) create mode 100644 docs/guides/singlestore/backup/kubestash/application-level/examples/backupconfiguration.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/application-level/examples/backupstorage.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/application-level/examples/restoresession.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/application-level/examples/retentionpolicy.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/application-level/examples/sample-singlestore.yaml create mode 100644 docs/guides/singlestore/backup/kubestash/application-level/index.md diff --git a/docs/guides/singlestore/backup/kubestash/application-level/examples/backupconfiguration.yaml b/docs/guides/singlestore/backup/kubestash/application-level/examples/backupconfiguration.yaml new file mode 100644 index 0000000000..4d9e1fbae3 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/application-level/examples/backupconfiguration.yaml @@ -0,0 +1,37 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: manifest-backup + - name: logical-backup \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/application-level/examples/backupstorage.yaml b/docs/guides/singlestore/backup/kubestash/application-level/examples/backupstorage.yaml new file mode 100644 index 0000000000..6ab3df02ac --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/application-level/examples/backupstorage.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/application-level/examples/restoresession.yaml b/docs/guides/singlestore/backup/kubestash/application-level/examples/restoresession.yaml new file mode 100644 index 0000000000..fa1fe6e950 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/application-level/examples/restoresession.yaml @@ -0,0 +1,21 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + manifestOptions: + restoreNamespace: dev + mySQL: + db: true + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore + - name: manifest-restore \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/application-level/examples/retentionpolicy.yaml b/docs/guides/singlestore/backup/kubestash/application-level/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..4591562860 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/application-level/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/application-level/examples/sample-singlestore.yaml b/docs/guides/singlestore/backup/kubestash/application-level/examples/sample-singlestore.yaml new file mode 100644 index 0000000000..a3510d8591 --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/application-level/examples/sample-singlestore.yaml @@ -0,0 +1,50 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: sample-singlestore + namespace: demo +spec: + version: "8.7.10" + topology: + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + storageType: Durable + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/application-level/index.md b/docs/guides/singlestore/backup/kubestash/application-level/index.md new file mode 100644 index 0000000000..a172c3eb5c --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/application-level/index.md @@ -0,0 +1,804 @@ +--- +title: Application Level Backup & Restore SingleStore | KubeStash +description: Application Level Backup and Restore using KubeStash +menu: + docs_{{ .version }}: + identifier: guides-application-level-backup-stashv2 + name: Application Level Backup + parent: guides-sdb-backup-stashv2 + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Application Level Backup and Restore SingleStore database using KubeStash + +KubeStash offers application-level backup and restore functionality for `SingleStore` databases. It captures both manifest and logical data backups of any `SingleStore` database in a single snapshot. During the restore process, KubeStash first applies the `SingleStore` manifest to the cluster and then restores the data into it. + +This guide will give you how you can take application-level backup and restore your `SingleStore` databases using `Kubestash`. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. +- Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). +- Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). +- Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- If you are not familiar with how KubeStash backup and restore SingleStore databases, please check the following guide [here](/docs/guides/singlestore/backup/kubestash/overview/index.md). + +You should be familiar with the following `KubeStash` concepts: + +- [BackupStorage](https://kubestash.com/docs/latest/concepts/crds/backupstorage/) +- [BackupConfiguration](https://kubestash.com/docs/latest/concepts/crds/backupconfiguration/) +- [BackupSession](https://kubestash.com/docs/latest/concepts/crds/backupsession/) +- [RestoreSession](https://kubestash.com/docs/latest/concepts/crds/restoresession/) +- [Addon](https://kubestash.com/docs/latest/concepts/crds/addon/) +- [Function](https://kubestash.com/docs/latest/concepts/crds/function/) +- [Task](https://kubestash.com/docs/latest/concepts/crds/addon/#task-specification) + +To keep everything isolated, we are going to use a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> **Note:** YAML files used in this tutorial are stored in [docs/guides/singlestore/backup/kubestash/application-level/examples](docs/guides/singlestore/backup/kubestash/application-level/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. + +## Backup SingleStore + +KubeStash supports backups for `SingleStore` instances across different configurations, including Standalone, and Cluster setups. In this demonstration, we'll focus on a `SingleStore` database using singlestore clustering. The backup and restore process is similar for Standalone and Cluster configurations as well. + +This section will demonstrate how to take application-level backup of a `SingleStore` database. Here, we are going to deploy a `SingleStore` database using KubeDB. Then, we are going to back up the database at the application level to a `GCS` bucket. Finally, we will restore the entire `SingleStore` database. + +### Create SingleStore License Secret + +We need SingleStore License to create SingleStore Database. So, Ensure that you have acquired a license and then simply pass the license by secret. + +```bash +$ kubectl create secret generic -n demo license-secret \ + --from-literal=username=license \ + --from-literal=password='your-license-set-here' +secret/license-secret created +``` + +### Deploy Sample SingleStore Database + +Let's deploy a sample `SingleStore` database and insert some data into it. + +**Create SingleStore CR:** + +Below is the YAML of a sample `SingleStore` CR that we are going to create for this tutorial: + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: sample-singlestore + namespace: demo +spec: + version: "8.7.10" + topology: + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + storageType: Durable + deletionPolicy: WipeOut +``` + +Here, +- `.spec.topology` specifies about the clustering configuration of MySQL. +- `.Spec.topology.mode` specifies the mode of MySQL Cluster. During the demonstration we consider to use `GroupReplication`. + +Create the above `MySQL` CR, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/sample-mysql.yaml +mysql.kubedb.com/sample-mysql created +``` + +KubeDB will deploy a MySQL database according to the above specification. It will also create the necessary Secrets and Services to access the database. + +Let's check if the database is ready to use, + +```bash +$ kubectl get mysqls.kubedb.com -n demo +NAME VERSION STATUS AGE +sample-mysql 8.2.0 Ready 4m22s +``` + +The database is `Ready`. Verify that KubeDB has created a `Secret` and a `Service` for this database using the following commands, + +```bash +$ kubectl get secret -n demo +NAME TYPE DATA AGE +sample-mysql-auth Opaque 2 4m58s + +$ kubectl get service -n demo -l=app.kubernetes.io/instance=sample-mysql +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +sample-mysql ClusterIP 10.96.55.61 3306/TCP 97s +sample-mysql-pods ClusterIP None 3306/TCP 97s +sample-mysql-standby ClusterIP 10.96.211.186 3306/TCP 97 +``` + +Here, we have to use service `sample-mysql` and secret `sample-mysql-auth` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/mysql/concepts/appbinding/index.md) CR that holds the necessary information to connect with the database. + +**Verify AppBinding:** + +Verify that the `AppBinding` has been created successfully using the following command, + +```bash +$ kubectl get appbindings -n demo +NAME AGE +sample-mysql 9m24s +``` + +Let's check the YAML of the above `AppBinding`, + +```bash +$ kubectl get appbindings -n demo sample-mysql -o yaml +``` + +```yaml +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: sample-mysql + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: mysqls.kubedb.com + name: sample-mysql + namespace: demo + ownerReferences: + - apiVersion: kubedb.com/v1 + blockOwnerDeletion: true + controller: true + kind: MySQL + name: sample-mysql + uid: edde3e8b-7775-4f91-85a9-4ba4b96315f7 + resourceVersion: "5126" + uid: 86c9a149-f8ab-44c4-947f-5f9b402aad6c +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo + clientConfig: + service: + name: sample-mysql + path: / + port: 3306 + scheme: tcp + url: tcp(sample-mysql.demo.svc:3306)/ + ... + ... + secret: + name: sample-mysql-auth + type: kubedb.com/mysql + version: 8.2.0 +``` + +KubeStash uses the `AppBinding` CR to connect with the target database. It requires the following two fields to set in AppBinding's `.spec` section. + +- `.spec.clientConfig.service.name` specifies the name of the Service that connects to the database. +- `.spec.secret` specifies the name of the Secret that holds necessary credentials to access the database. +- `spec.type` specifies the types of the app that this AppBinding is pointing to. KubeDB generated AppBinding follows the following format: `/`. + +**Insert Sample Data:** + +Now, we are going to exec into the database pod and create some sample data. At first, find out the database Pod using the following command, + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-mysql" +NAME READY STATUS RESTARTS AGE +sample-mysql-0 2/2 Running 0 33m +sample-mysql-1 2/2 Running 0 33m +sample-mysql-2 2/2 Running 0 33m +``` + +And copy the username and password of the `root` user to access into `mysql` shell. + +```bash +$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.username}'| base64 -d +root⏎ + +$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.password}'| base64 -d +DZfmUZd14fNEEOU4⏎ +``` + +Now, Lets exec into the Pod to enter into `mysql` shell and create a database and a table, + +```bash +$ kubectl exec -it -n demo sample-mysql-0 -- mysql --user=root --password=DZfmUZd14fNEEOU4 +Defaulted container "mysql" out of: mysql, mysql-init (init) +mysql: [Warning] Using a password on the command line interface can be insecure. +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 977 +Server version: 8.2.0 MySQL Community Server - GPL + +Copyright (c) 2000, 2023, Oracle and/or its affiliates. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> CREATE DATABASE playground; +Query OK, 1 row affected (0.01 sec) + +mysql> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| information_schema | +| mysql | +| performance_schema | +| playground | +| sys | ++--------------------+ +5 rows in set (0.00 sec) + +mysql> CREATE TABLE playground.equipment ( id INT NOT NULL AUTO_INCREMENT, type VARCHAR(50), quant INT, color VARCHAR(25), PRIMARY KEY(id)); +Query OK, 0 rows affected (0.01 sec) + +mysql> SHOW TABLES IN playground; ++----------------------+ +| Tables_in_playground | ++----------------------+ +| equipment | ++----------------------+ +1 row in set (0.01 sec) + +mysql> INSERT INTO playground.equipment (type, quant, color) VALUES ("slide", 2, "blue"); +Query OK, 1 row affected (0.01 sec) + +mysql> SELECT * FROM playground.equipment; ++----+-------+-------+-------+ +| id | type | quant | color | ++----+-------+-------+-------+ +| 1 | slide | 2 | blue | ++----+-------+-------+-------+ +1 row in set (0.00 sec) + +mysql> exit +Bye +``` +Now, we are ready to backup the database. + +### Prepare Backend + +We are going to store our backed up data into a GCS bucket. We have to create a Secret with necessary credentials and a `BackupStorage` CR to use this backend. If you want to use a different backend, please read the respective backend configuration doc from [here](https://kubestash.com/docs/latest/guides/backends/overview/). + +**Create Secret:** + +Let's create a secret called `gcs-secret` with access credentials to our desired GCS bucket, + +```bash +$ echo -n '' > GOOGLE_PROJECT_ID +$ cat /path/to/downloaded-sa-key.json > GOOGLE_SERVICE_ACCOUNT_JSON_KEY +$ kubectl create secret generic -n demo gcs-secret \ + --from-file=./GOOGLE_PROJECT_ID \ + --from-file=./GOOGLE_SERVICE_ACCOUNT_JSON_KEY +secret/gcs-secret created +``` + +**Create BackupStorage:** + +Now, create a `BackupStorage` using this secret. Below is the YAML of `BackupStorage` CR we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: gcs-storage + namespace: demo +spec: + storage: + provider: gcs + gcs: + bucket: kubestash-qa + prefix: demo + secretName: gcs-secret + usagePolicy: + allowedNamespaces: + from: All + default: true + deletionPolicy: Delete +``` + +Let's create the BackupStorage we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/backupstorage.yaml +backupstorage.storage.kubestash.com/gcs-storage created +``` + +Now, we are ready to backup our database to our desired backend. + +**Create RetentionPolicy:** + +Now, let's create a `RetentionPolicy` to specify how the old Snapshots should be cleaned up. + +Below is the YAML of the `RetentionPolicy` object that we are going to create, + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Let’s create the above `RetentionPolicy`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/retentionpolicy.yaml +retentionpolicy.storage.kubestash.com/demo-retention created +``` + +### Backup + +We have to create a `BackupConfiguration` targeting respective `sample-mysql` MySQL database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. + +At first, we need to create a secret with a Restic password for backup data encryption. + +**Create Secret:** + +Let's create a secret called `encrypt-secret` with the Restic password, + +```bash +$ echo -n 'changeit' > RESTIC_PASSWORD +$ kubectl create secret generic -n demo encrypt-secret \ + --from-file=./RESTIC_PASSWORD \ +secret "encrypt-secret" created +``` + +**Create BackupConfiguration:** + +Below is the YAML for `BackupConfiguration` CR to take application-level backup of the `sample-mysql` database that we have deployed earlier, + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: sample-mysql-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: MySQL + namespace: demo + name: sample-mysql + backends: + - name: gcs-backend + storageRef: + namespace: demo + name: gcs-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: gcs-mysql-repo + backend: gcs-backend + directory: /mysql + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: manifest-backup + - name: logical-backup +``` + +- `.spec.sessions[*].schedule` specifies that we want to backup at `5 minutes` interval. +- `.spec.target` refers to the targeted `sample-mysql` MySQL database that we created earlier. +- `.spec.sessions[*].addon.tasks[*].name[*]` specifies that both the `manifest-backup` and `logical-backup` tasks will be executed. + +Let's create the `BackupConfiguration` CR that we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/backupconfiguration.yaml +backupconfiguration.core.kubestash.com/sample-mysql-backup created +``` + +**Verify Backup Setup Successful** + +If everything goes well, the phase of the `BackupConfiguration` should be `Ready`. The `Ready` phase indicates that the backup setup is successful. Let's verify the `Phase` of the BackupConfiguration, + +```bash +$ kubectl get backupconfiguration -n demo +NAME PHASE PAUSED AGE +sample-mysql-backup Ready 2m50s +``` + +Additionally, we can verify that the `Repository` specified in the `BackupConfiguration` has been created using the following command, + +```bash +$ kubectl get repo -n demo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-mysql-repo 0 0 B Ready 3m +``` + +KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/mysql` directory. + +**Verify CronJob:** + +It will also create a `CronJob` with the schedule specified in `spec.sessions[*].scheduler.schedule` field of `BackupConfiguration` CR. + +Verify that the `CronJob` has been created using the following command, + +```bash +$ kubectl get cronjob -n demo +NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE +trigger-sample-mysql-backup-frequent-backup */5 * * * * 0 2m45s 3m25s +``` + +**Verify BackupSession:** + +KubeStash triggers an instant backup as soon as the `BackupConfiguration` is ready. After that, backups are scheduled according to the specified schedule. + +Run the following command to watch `BackupSession` CR, + +```bash +$ kubectl get backupsession -n demo -w + +NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE +sample-mysql-backup-frequent-backup-1724065200 BackupConfiguration sample-mysql-backup Succeeded 7m22s +``` + +We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. + +**Verify Backup:** + +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-mysql-backup` has been updated by the following command, + +```bash +$ kubectl get repository -n demo gcs-mysql-repo +NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE +gcs-mysql-repo true 1 806 B Ready 8m27s 9m18s +``` + +At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. + +```bash +$ kubectl get snapshots -n demo -l=kubestash.com/repo-name=gcs-demo-repo +NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE +gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 sample-mysql-backup frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +``` + +> Note: KubeStash creates a `Snapshot` with the following labels: +> - `kubestash.com/app-ref-kind: ` +> - `kubestash.com/app-ref-name: ` +> - `kubestash.com/app-ref-namespace: ` +> - `kubestash.com/repo-name: ` +> +> These labels can be used to watch only the `Snapshot`s related to our target Database or `Repository`. + +If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. + +```bash +$ kubectl get snapshots -n demo gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 -oyaml +``` + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: Snapshot +metadata: + creationTimestamp: "2024-09-03T10:25:00Z" + finalizers: + - kubestash.com/cleanup + generation: 1 + labels: + kubestash.com/app-ref-kind: MySQL + kubestash.com/app-ref-name: sample-mysql + kubestash.com/app-ref-namespace: demo + kubestash.com/repo-name: gcs-mysql-repo + annotations: + kubedb.com/db-version: 8.2.0 + name: gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 + namespace: demo + ownerReferences: + - apiVersion: storage.kubestash.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: Repository + name: gcs-mysql-repo + uid: 1f5ba355-7f99-4b99-8bbf-9f9d4f31c52a + resourceVersion: "213010" + uid: 18cabb10-e594-4655-8763-3daa0872508e +spec: + appRef: + apiGroup: kubedb.com + kind: MySQL + name: sample-mysql + namespace: demo + backupSession: sample-mysql-backup-frequent-backup-1725359100 + deletionPolicy: Delete + repository: gcs-mysql-repo + session: frequent-backup + snapshotID: 01J6VPN4TPHDFT1M9Q9YVGMTKF + type: FullBackup + version: v1 +status: + components: + dump: + driver: Restic + duration: 7.393324414s + integrity: true + path: repository/v1/frequent-backup/dump + phase: Succeeded + resticStats: + - hostPath: dumpfile.sql + id: f2ffd1bdb98563e15c46d8927d7239873ce7094132d959e12134688e06984736 + size: 3.657 MiB + uploaded: 706.081 KiB + size: 893.009 KiB + manifest: + driver: Restic + duration: 12.672292995s + integrity: true + path: repository/v1/frequent-backup/manifest + phase: Succeeded + resticStats: + - hostPath: /kubestash-tmp/manifest + id: ff99eb7ea769a365f7cdc83a252df610c262fc934ec0a3475499bbbb35ca6931 + size: 2.883 KiB + uploaded: 1.440 KiB + size: 3.788 KiB + conditions: + - lastTransitionTime: "2024-09-03T10:25:00Z" + message: Recent snapshot list updated successfully + reason: SuccessfullyUpdatedRecentSnapshotList + status: "True" + type: RecentSnapshotListUpdated + - lastTransitionTime: "2024-09-03T10:25:49Z" + message: Metadata uploaded to backend successfully + reason: SuccessfullyUploadedSnapshotMetadata + status: "True" + type: SnapshotMetadataUploaded + integrity: true + phase: Succeeded + size: 896.796 KiB + snapshotTime: "2024-09-03T10:25:00Z" + totalComponents: 2 +``` + +> KubeStash uses the `mysqldump` command to take backups of target MySQL databases. Therefore, the component name for logical backups is set as `dump`. +> KubeStash set component name as `manifest` for the `manifest backup` of MySQL databases. + +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/mysql/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/dep/snapshots` directory. + +> Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. + +## Restore + +In this section, we are going to restore the entire database from the backup that we have taken in the previous section. + +For this tutorial, we will restore the database in a separate namespace called `dev`. + +First, create the namespace by running the following command: + +```bash +$ kubectl create ns dev +namespace/dev created +``` + +#### Create RestoreSession: + +We need to create a RestoreSession CR. + +Below, is the contents of YAML file of the `RestoreSession` CR that we are going to create to restore the entire database. + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: restore-sample-mysql + namespace: demo +spec: + manifestOptions: + restoreNamespace: dev + mySQL: + db: true + dataSource: + repository: gcs-mysql-repo + snapshot: latest + encryptionSecret: + name: encrypt-secret + namespace: demo + addon: + name: mysql-addon + tasks: + - name: logical-backup-restore + - name: manifest-restore +``` + +Here, + +- `.spec.manifestOptions.mySQL.db` specifies whether to restore the DB manifest or not. +- `.spec.dataSource.repository` specifies the Repository object that holds the backed up data. +- `.spec.dataSource.snapshot` specifies to restore from latest `Snapshot`. +- `.spec.addon.tasks[*]` specifies that both the `manifest-restore` and `logical-backup-restore` tasks. + +Let's create the RestoreSession CRD object we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/restoresession.yaml +restoresession.core.kubestash.com/sample-mysql-restore created +``` + +Once, you have created the `RestoreSession` object, KubeStash will create restore Job. Run the following command to watch the phase of the `RestoreSession` object, + +```bash +$ watch kubectl get restoresession -n demo +Every 2.0s: kubectl get restores... AppsCode-PC-03: Wed Aug 21 10:44:05 2024 + +NAME REPOSITORY FAILURE-POLICY PHASE DURATION AGE +sample-restore gcs-demo-repo Succeeded 3s 53s +``` +The `Succeeded` phase means that the restore process has been completed successfully. + +#### Verify Restored MySQL Manifest: + +In this section, we will verify whether the desired `MySQL` database manifest has been successfully applied to the cluster. + +```bash +$ kubectl get mysqls.kubedb.com -n dev +NAME VERSION STATUS AGE +sample-mysql 8.2.0 Ready 39m +``` + +The output confirms that the `MySQL` database has been successfully created with the same configuration as it had at the time of backup. + +#### Verify Restored Data: + +In this section, we are going to verify whether the desired data has been restored successfully. We are going to connect to the database server and check whether the database and the table we created earlier in the original database are restored. + +At first, check if the database has gone into `Ready` state by the following command, + +```bash +$ kubectl get my -n dev sample-mysql +NAME VERSION STATUS AGE +sample-mysql 8.2.0 Ready 4m +``` + +Now, find out the database `Pod` by the following command, + +```bash +$ kubectl get pods -n dev --selector="app.kubernetes.io/instance=sample-mysql" +NAME READY STATUS RESTARTS AGE +sample-mysql-0 2/2 Running 0 2m +sample-mysql-1 2/2 Running 0 2m +sample-mysql-2 2/2 Running 0 2m +``` + +And then copy the username and password of the `root` user to access into `mysql` shell. + +```bash +$ kubectl get secret -n dev sample-mysql-auth -o jsonpath='{.data.username}'| base64 -d +root + +$ kubectl get secret -n dev sample-mysql-auth -o jsonpath='{.data.password}'| base64 -d +QMm1hi0T*7QFz_yh +``` + +```bash +$ kubectl exec -it -n dev sample-mysql-0 -- mysql --user=root --password='QMm1hi0T*7QFz_yh' +Defaulted container "mysql" out of: mysql, mysql-coordinator, mysql-init (init) +mysql: [Warning] Using a password on the command line interface can be insecure. +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 243 +Server version: 8.2.0 MySQL Community Server - GPL + +Copyright (c) 2000, 2023, Oracle and/or its affiliates. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| information_schema | +| mysql | +| performance_schema | +| playground | +| sys | ++--------------------+ +5 rows in set (0.00 sec) + +mysql> SHOW TABLES IN playground; ++----------------------+ +| Tables_in_playground | ++----------------------+ +| equipment | ++----------------------+ +1 row in set (0.00 sec) + +mysql> SELECT * FROM playground.equipment; ++----+-------+-------+-------+ +| id | type | quant | color | ++----+-------+-------+-------+ +| 1 | slide | 2 | blue | ++----+-------+-------+-------+ +1 row in set (0.00 sec) + +mysql> exit +Bye +``` + +So, from the above output, we can see that the `playground` database and the `equipment` table we have created earlier in the original database and now, they are restored successfully. + +## Cleanup + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl delete backupconfigurations.core.kubestash.com -n demo sample-mysql-backup +kubectl delete backupstorage -n demo gcs-storage +kubectl delete secret -n demo gcs-secret +kubectl delete secret -n demo encrypt-secret +kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention +kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-mysql +kubectl delete my -n demo sample-mysql +kubectl delete my -n dev sample-mysql +``` \ No newline at end of file From 01c85fa9154ea1bd77312cb86af6ca9e1f4f8fd2 Mon Sep 17 00:00:00 2001 From: AshrafulHaqueToni Date: Fri, 20 Sep 2024 09:57:41 +0600 Subject: [PATCH 08/10] extra part Signed-off-by: AshrafulHaqueToni --- .../backup/kubestash/logical/index.md | 312 +++++++++++------- 1 file changed, 192 insertions(+), 120 deletions(-) diff --git a/docs/guides/singlestore/backup/kubestash/logical/index.md b/docs/guides/singlestore/backup/kubestash/logical/index.md index 91de127770..5ab08c5c30 100644 --- a/docs/guides/singlestore/backup/kubestash/logical/index.md +++ b/docs/guides/singlestore/backup/kubestash/logical/index.md @@ -44,77 +44,128 @@ namespace/demo created > **Note:** YAML files used in this tutorial are stored in [docs/guides/mysql/backup/kubestash/logical/examples](docs/guides/mysql/backup/kubestash/logical/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. -## Backup MySQL +## Backup SingleStore -KubeStash supports backups for `MySQL` instances across different configurations, including Standalone, Group Replication, and InnoDB Cluster setups. In this demonstration, we'll focus on a `MySQL` database using Group Replication. The backup and restore process is similar for Standalone and InnoDB Cluster configurations as well. +KubeStash supports backups for `MySQL` instances across different configurations, including Standalone, and Cluster setups. In this demonstration, we'll focus on a `SingleStore` database using Clustering mode. The backup and restore process is similar for Standalone and Cluster configurations as well. -This section will demonstrate how to backup a `MySQL` database. Here, we are going to deploy a `MySQL` database using KubeDB. Then, we are going to backup this database into a `GCS` bucket. Finally, we are going to restore the backup up data into another `MySQL` database. +This section will demonstrate how to backup a `SingleStore` database. Here, we are going to deploy a `SingleStore` database using KubeDB. Then, we are going to backup this database into a `GCS` bucket. Finally, we are going to restore the backup up data into another `SingleStore` database. -### Deploy Sample MySQL Database +### Create SingleStore License Secret -Let's deploy a sample `MySQL` database and insert some data into it. +We need SingleStore License to create SingleStore Database. So, Ensure that you have acquired a license and then simply pass the license by secret. -**Create MySQL CR:** +```bash +$ kubectl create secret generic -n demo license-secret \ + --from-literal=username=license \ + --from-literal=password='your-license-set-here' +secret/license-secret created +``` + + +### Deploy Sample SingleStore Database + +Let's deploy a sample `SingleStore` database and insert some data into it. + +**Create SingleStore CR:** -Below is the YAML of a sample `MySQL` CRD that we are going to create for this tutorial: +Below is the YAML of a sample `SingleStore` CRD that we are going to create for this tutorial: ```yaml -apiVersion: kubedb.com/v1 -kind: MySQL +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore metadata: - name: sample-mysql + name: sdb-sample namespace: demo spec: - version: "8.2.0" - replicas: 3 + version: "8.7.10" topology: - mode: GroupReplication + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret storageType: Durable - storage: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Mi deletionPolicy: WipeOut ``` Here, -- `.spec.topology` specifies about the clustering configuration of MySQL. -- `.Spec.topology.mode` specifies the mode of MySQL Cluster. During the demonstration we consider to use `GroupReplication`. -Create the above `MySQL` CR, +- `spec.version` is the name of the SinglestoreVersion CRD where the docker images are specified. In this tutorial, a SingleStore `8.7.10` database is going to be created. +- `spec.topology` specifies that it will be used as cluster mode. If this field is nil it will be work as standalone mode. +- `spec.topology.aggregator.replicas` or `spec.topology.leaf.replicas` specifies that the number replicas that will be used for aggregator or leaf. +- `spec.storageType` specifies the type of storage that will be used for SingleStore database. It can be `Durable` or `Ephemeral`. Default value of this field is `Durable`. If `Ephemeral` is used then KubeDB will create SingleStore database using `EmptyDir` volume. In this case, you don't have to specify `spec.storage` field. This is useful for testing purposes. +- `spec.topology.aggregator.storage` or `spec.topology.leaf.storage` specifies the StorageClass of PVC dynamically allocated to store data for this database. This storage spec will be passed to the PetSet created by KubeDB operator to run database pods. You can specify any StorageClass available in your cluster with appropriate resource requests. +- `spec.deletionPolicy` gives flexibility whether to `nullify`(reject) the delete operation of `Singlestore` crd or which resources KubeDB should keep or delete when you delete `Singlestore` crd. If admission webhook is enabled, It prevents users from deleting the database as long as the `spec.deletionPolicy` is set to `DoNotTerminate`. Learn details of all `DeletionPolicy` [here](/docs/guides/mysql/concepts/database/index.md#specdeletionpolicy) + +> Note: `spec.storage` section is used to create PVC for database pod. It will create PVC with storage size specified in `storage.resources.requests` field. Don't specify limits here. PVC does not get resized automatically. + +Create the above `SingleStore` CR, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/sample-mysql.yaml -mysql.kubedb.com/sample-mysql created +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/logical/examples/sdb-sample.yaml +singlestore.kubedb.com/sdb-sample created ``` -KubeDB will deploy a MySQL database according to the above specification. It will also create the necessary `Secrets` and `Services` to access the database. +KubeDB will deploy a SingleStore database according to the above specification. It will also create the necessary `Secrets` and `Services` to access the database. Let's check if the database is ready to use, ```bash $ kubectl get mysqls.kubedb.com -n demo NAME VERSION STATUS AGE -sample-mysql 8.2.0 Ready 4m22s +sdb-sample 8.7.10 Ready 4m22s ``` The database is `Ready`. Verify that KubeDB has created a `Secret` and a `Service` for this database using the following commands, ```bash -$ kubectl get secret -n demo -l=app.kubernetes.io/instance=sample-mysql -NAME TYPE DATA AGE -sample-mysql-auth Opaque 2 4m58s +$ kubectl get secret -n demo -l=app.kubernetes.io/instance=sdb-sample +NAME TYPE DATA AGE +sdb-sample-root-cred kubernetes.io/basic-auth 2 4m58s + +$ kubectl get service -n demo -l=app.kubernetes.io/instance=sdb-sample +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +sdb-sample ClusterIP 10.128.230.168 3306/TCP,8081/TCP 5m10s +sdb-sample-pods ClusterIP None 3306/TCP 5m10s -$ kubectl get service -n demo -l=app.kubernetes.io/instance=sample-mysql -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -sample-mysql ClusterIP 10.96.55.61 3306/TCP 97s -sample-mysql-pods ClusterIP None 3306/TCP 97s -sample-mysql-standby ClusterIP 10.96.211.186 3306/TCP 97 ``` -Here, we have to use service `sample-mysql` and secret `sample-mysql-auth` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/mysql/concepts/appbinding/index.md) CR that holds the necessary information to connect with the database. +Here, we have to use service `sdb-sample` and secret `sdb-sample-root-cred` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/singlestore/concepts/appbinding/index.md) CR that holds the necessary information to connect with the database. **Verify AppBinding:** @@ -123,95 +174,112 @@ Verify that the `AppBinding` has been created successfully using the following c ```bash $ kubectl get appbindings -n demo NAME AGE -sample-mysql 9m24s +sdb-sample 9m24s ``` Let's check the YAML of the above `AppBinding`, ```bash -$ kubectl get appbindings -n demo sample-mysql -o yaml +$ kubectl get appbindings -n demo sdb-sample -o yaml ``` ```yaml apiVersion: appcatalog.appscode.com/v1alpha1 kind: AppBinding metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"kubedb.com/v1alpha2","kind":"Singlestore","metadata":{"annotations":{},"name":"sdb-sample","namespace":"demo"},"spec":{"deletionPolicy":"WipeOut","licenseSecret":{"name":"license-secret"},"storageType":"Durable","topology":{"aggregator":{"podTemplate":{"spec":{"containers":[{"name":"singlestore","resources":{"limits":{"cpu":"0.6","memory":"2Gi"},"requests":{"cpu":"0.6","memory":"2Gi"}}}]}},"replicas":2,"storage":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"1Gi"}}}},"leaf":{"podTemplate":{"spec":{"containers":[{"name":"singlestore","resources":{"limits":{"cpu":"0.6","memory":"2Gi"},"requests":{"cpu":"0.6","memory":"2Gi"}}}]}},"replicas":3,"storage":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}}}},"version":"8.7.10"}} + creationTimestamp: "2024-09-11T07:03:44Z" + generation: 1 labels: app.kubernetes.io/component: database - app.kubernetes.io/instance: sample-mysql + app.kubernetes.io/instance: sdb-sample app.kubernetes.io/managed-by: kubedb.com - app.kubernetes.io/name: mysqls.kubedb.com - name: sample-mysql + app.kubernetes.io/name: singlestores.kubedb.com + name: sdb-sample namespace: demo ownerReferences: - - apiVersion: kubedb.com/v1 + - apiVersion: kubedb.com/v1alpha2 blockOwnerDeletion: true controller: true - kind: MySQL - name: sample-mysql - uid: edde3e8b-7775-4f91-85a9-4ba4b96315f7 - resourceVersion: "5126" - uid: 86c9a149-f8ab-44c4-947f-5f9b402aad6c + kind: Singlestore + name: sdb-sample + uid: e08e1f37-d869-437d-9b15-14c6aef3f406 + resourceVersion: "4904220" + uid: 92b2b318-1874-4471-97ec-d789c6e16809 spec: appRef: apiGroup: kubedb.com - kind: MySQL - name: sample-mysql + kind: Singlestore + name: sdb-sample namespace: demo clientConfig: service: - name: sample-mysql + name: sdb-sample path: / port: 3306 scheme: tcp - url: tcp(sample-mysql.demo.svc:3306)/ - ... - ... + url: tcp(sdb-sample.demo.svc:3306)/ + parameters: + apiVersion: config.kubedb.com/v1alpha1 + kind: SinglestoreConfiguration + masterAggregator: sdb-sample-aggregator-0.sdb-sample-pods.demo.svc + stash: + addon: + backupTask: + name: "" + restoreTask: + name: "" secret: - name: sample-mysql-auth - type: kubedb.com/mysql - version: 8.2.0 + name: sdb-sample-root-cred + type: kubedb.com/singlestore + version: 8.7.10 + ``` KubeStash uses the `AppBinding` CR to connect with the target database. It requires the following two fields to set in AppBinding's `.spec` section. +- `.spec.parameters.masterAggregator` specifies the dns of master aggregator node that we have to mention in mysqldump command when taken backup or restore. - `.spec.clientConfig.service.name` specifies the name of the Service that connects to the database. - `.spec.secret` specifies the name of the Secret that holds necessary credentials to access the database. - `spec.type` specifies the types of the app that this AppBinding is pointing to. KubeDB generated AppBinding follows the following format: `/`. **Insert Sample Data:** -Now, we are going to exec into the database pod and create some sample data. At first, find out the database `Pod` using the following command, +Now, we are going to exec into the any aggregator pod and create some sample data. At first, find out the database `Pod` using the following command, ```bash -$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-mysql" -NAME READY STATUS RESTARTS AGE -sample-mysql-0 2/2 Running 0 2m41s -sample-mysql-1 2/2 Running 0 2m35s -sample-mysql-2 2/2 Running 0 2m29s +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sdb-sample" +NAME READY STATUS RESTARTS AGE +sdb-sample-aggregator-0 2/2 Running 0 15m +sdb-sample-aggregator-1 2/2 Running 0 15m +sdb-sample-leaf-0 2/2 Running 0 15m +sdb-sample-leaf-1 2/2 Running 0 15m +sdb-sample-leaf-2 2/2 Running 0 15m ``` -And copy the username and password of the `root` user to access into `mysql` shell. +And copy the username and password of the `root` user to access into `memsql` shell. ```bash -$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.username}'| base64 -d -root⏎ +$ kubectl get secret -n demo sdb-sample-root-cred -o jsonpath='{.data.username}'| base64 -d +root⏎ -$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.password}'| base64 -d -DZfmUZd14fNEEOU4⏎ +kubectl get secret -n demo sdb-sample-root-cred -o jsonpath='{.data.password}'| base64 -d +xEJv73q3w_m1~H.G⏎ ``` -Now, Lets exec into the `Pod` to enter into `mysql` shell and create a database and a table, +Now, Lets exec into the any aggregator `Pod` to enter into `memsql` shell and create a database and a table, ```bash -$ kubectl exec -it -n demo sample-mysql-0 -- mysql --user=root --password=DZfmUZd14fNEEOU4 -Defaulted container "mysql" out of: mysql, mysql-init (init) -mysql: [Warning] Using a password on the command line interface can be insecure. +$ kubectl exec -it -n demo sdb-sample-aggregator-0 -- singlestore --user=root --password=xEJv73q3w_m1~H.G +Defaulted container "singlestore" out of: singlestore, singlestore-coordinator, singlestore-init (init) +singlestore-client: [Warning] Using a password on the command line interface can be insecure. Welcome to the MySQL monitor. Commands end with ; or \g. -Your MySQL connection id is 977 -Server version: 8.2.0 MySQL Community Server - GPL +Your MySQL connection id is 300070 +Server version: 5.7.32 SingleStoreDB source distribution (compatible; MySQL Enterprise & MySQL Commercial) -Copyright (c) 2000, 2023, Oracle and/or its affiliates. +Copyright (c) 2000, 2022, Oracle and/or its affiliates. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective @@ -219,45 +287,49 @@ owners. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. -mysql> CREATE DATABASE playground; +singlestore> CREATE DATABASE playground partitions 2; Query OK, 1 row affected (0.01 sec) -mysql> SHOW DATABASES; +singlestore> show databases; +--------------------+ | Database | +--------------------+ +| cluster | +| demo | +| det | | information_schema | -| mysql | -| performance_schema | +| memsql | | playground | -| sys | +| singlestore_health | +| test | +--------------------+ -5 rows in set (0.00 sec) +8 rows in set (0.00 sec) -mysql> CREATE TABLE playground.equipment ( id INT NOT NULL AUTO_INCREMENT, type VARCHAR(50), quant INT, color VARCHAR(25), PRIMARY KEY(id)); -Query OK, 0 rows affected (0.01 sec) +singlestore> CREATE TABLE playground.equipment ( id INT NOT NULL AUTO_INCREMENT, type VARCHAR(50), quant INT, color VARCHAR(25), PRIMARY KEY(id)); +Query OK, 0 rows affected, 1 warning (0.27 sec) -mysql> SHOW TABLES IN playground; +singlestore> SHOW TABLES IN playground; +----------------------+ | Tables_in_playground | +----------------------+ | equipment | +----------------------+ -1 row in set (0.01 sec) +1 row in set (0.00 sec) -mysql> INSERT INTO playground.equipment (type, quant, color) VALUES ("slide", 2, "blue"); -Query OK, 1 row affected (0.01 sec) +singlestore> INSERT INTO playground.equipment (type, quant, color) VALUES ("slide", 2, "blue"); +Query OK, 1 row affected (1.15 sec) -mysql> SELECT * FROM playground.equipment; +singlestore> SELECT * FROM playground.equipment; +----+-------+-------+-------+ | id | type | quant | color | +----+-------+-------+-------+ | 1 | slide | 2 | blue | +----+-------+-------+-------+ -1 row in set (0.00 sec) +1 row in set (0.14 sec) -mysql> exit +singlestore> exit Bye + ``` Now, we are ready to backup the database. @@ -345,7 +417,7 @@ retentionpolicy.storage.kubestash.com/demo-retention created ### Backup -We have to create a `BackupConfiguration` targeting respective `sample-mysql` MySQL database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. +We have to create a `BackupConfiguration` targeting respective `sdb-sample` SingleStore database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. At first, we need to create a secret with a Restic password for backup data encryption. @@ -362,20 +434,20 @@ secret "encrypt-secret" created **Create BackupConfiguration:** -Below is the YAML for `BackupConfiguration` CR to backup the `sample-mysql` database that we have deployed earlier, +Below is the YAML for `BackupConfiguration` CR to backup the `sdb-sample` database that we have deployed earlier, ```yaml apiVersion: core.kubestash.com/v1alpha1 kind: BackupConfiguration metadata: - name: sample-mysql-backup + name: sample-sdb-backup namespace: demo spec: target: apiGroup: kubedb.com - kind: MySQL + kind: Singlestore namespace: demo - name: sample-mysql + name: sdb-sample backends: - name: gcs-backend storageRef: @@ -391,26 +463,26 @@ spec: jobTemplate: backoffLimit: 1 repositories: - - name: gcs-mysql-repo + - name: gcs-sdb-repo backend: gcs-backend - directory: /mysql + directory: /singlestore encryptionSecret: name: encrypt-secret namespace: demo addon: - name: mysql-addon + name: singlestore-addon tasks: - name: logical-backup ``` - `.spec.sessions[*].schedule` specifies that we want to backup the database at `5 minutes` interval. -- `.spec.target` refers to the targeted `sample-mysql` MySQL database that we created earlier. +- `.spec.target` refers to the targeted `sdb-sample` SigleStore database that we created earlier. Let's create the `BackupConfiguration` CR that we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/backupconfiguration.yaml -backupconfiguration.core.kubestash.com/sample-mysql-backup created +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/logical/examples/backupconfiguration.yaml +backupconfiguration.core.kubestash.com/sample-sdb-backup created ``` **Verify Backup Setup Successful** @@ -420,7 +492,7 @@ If everything goes well, the phase of the `BackupConfiguration` should be `Ready ```bash $ kubectl get backupconfiguration -n demo NAME PHASE PAUSED AGE -sample-mysql-backup Ready 2m50s +sample-sdb-backup Ready 2m50s ``` Additionally, we can verify that the `Repository` specified in the `BackupConfiguration` has been created using the following command, @@ -428,10 +500,10 @@ Additionally, we can verify that the `Repository` specified in the `BackupConfig ```bash $ kubectl get repo -n demo NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE -gcs-mysql-repo 0 0 B Ready 3m +gcs-sdb-repo 0 0 B Ready 3m ``` -KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/mysql` directory. +KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/singlestore` directory. **Verify CronJob:** @@ -442,7 +514,7 @@ Verify that the `CronJob` has been created using the following command, ```bash $ kubectl get cronjob -n demo NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE -trigger-sample-mysql-backup-frequent-backup */5 * * * * 0 2m45s 3m25s +trigger-sample-sdb-backup-frequent-backup */5 * * * * 0 2m45s 3m25s ``` **Verify BackupSession:** @@ -453,19 +525,19 @@ KubeStash triggers an instant backup as soon as the `BackupConfiguration` is rea $ kubectl get backupsession -n demo -w NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE -sample-mysql-backup-frequent-backup-1724065200 BackupConfiguration sample-mysql-backup Succeeded 7m22s +sample-sdb-backup-frequent-backup-1724065200 BackupConfiguration sample-mysql-backup Succeeded 7m22s ``` We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. **Verify Backup:** -Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-mysql-backup` has been updated by the following command, +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-sdb-backup` has been updated by the following command, ```bash -$ kubectl get repository -n demo gcs-mysql-repo +$ kubectl get repository -n demo gcs-sdb-repo NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE -gcs-mysql-repo true 1 806 B Ready 8m27s 9m18s +gcs-sdb-repo true 1 806 B Ready 8m27s 9m18s ``` At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. @@ -473,7 +545,7 @@ At this moment we have one `Snapshot`. Run the following command to check the re ```bash $ kubectl get snapshots -n demo -l=kubestash.com/repo-name=gcs-demo-repo NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE -gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 sample-mysql-backup frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +gcs-sdb-repo-sample-sdb-backup-frequent-backup-1724065200 sample-sdb-backup frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h ``` > Note: KubeStash creates a `Snapshot` with the following labels: @@ -487,7 +559,7 @@ gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 sample-mysql-bac If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. ```bash -$ kubectl get snapshots -n demo gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 -oyaml +$ kubectl get snapshots -n demo gcs-sdb-repo-sample-sdb-backup-frequent-backup-1724065200 -oyaml ``` ```yaml @@ -499,32 +571,32 @@ metadata: - kubestash.com/cleanup generation: 1 labels: - kubestash.com/app-ref-kind: MySQL - kubestash.com/app-ref-name: sample-mysql + kubestash.com/app-ref-kind: Singlestore + kubestash.com/app-ref-name: sdb-sample kubestash.com/app-ref-namespace: demo - kubestash.com/repo-name: gcs-mysql-repo + kubestash.com/repo-name: gcs-sdb-repo annotations: - kubedb.com/db-version: 8.2.0 - name: gcs-mysql-repo-sample-mysql-backup-frequent-backup-1724065200 + kubedb.com/db-version: 8.7.10 + name: gcs-sdb-repo-sample-sdb-backup-frequent-backup-1724065200 namespace: demo ownerReferences: - apiVersion: storage.kubestash.com/v1alpha1 blockOwnerDeletion: true controller: true kind: Repository - name: gcs-mysql-repo + name: gcs-sdb-repo uid: 036a2605-0dcc-43bd-a921-363d5c3e8cf0 resourceVersion: "33742" uid: d7f14919-4c31-4b46-84e9-0a75f303ad92 spec: appRef: apiGroup: kubedb.com - kind: MySQL - name: sample-mysql + kind: Singlestore + name: sdb-sample namespace: demo - backupSession: sample-mysql-backup-frequent-backup-1724065200 + backupSession: sample-sdb-backup-frequent-backup-1724065200 deletionPolicy: Delete - repository: gcs-mysql-repo + repository: gcs-sdb-repo session: frequent-backup snapshotID: 01J6V4P4J0R33C8EG0JWK82118 type: FullBackup @@ -548,7 +620,7 @@ status: > KubeStash uses the `mysqldump` command to take backups of target MySQL databases. Therefore, the component name for logical backups is set as `dump`. -Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/mysql/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/dep/snapshots` directory. +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/singlestore/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/dep/snapshots` directory. > Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. From f5f62159d74fe90dfda30574258116751731ce1d Mon Sep 17 00:00:00 2001 From: Ashraful Haque Tani Date: Fri, 20 Sep 2024 11:53:58 +0600 Subject: [PATCH 09/10] completed logical backup Signed-off-by: Ashraful Haque Tani --- .../examples/restored-singlestore.yaml | 52 +++++ .../logical/examples/restoresession.yaml | 12 +- .../backup/kubestash/logical/index.md | 180 +++++++++++------- 3 files changed, 168 insertions(+), 76 deletions(-) create mode 100644 docs/guides/singlestore/backup/kubestash/logical/examples/restored-singlestore.yaml diff --git a/docs/guides/singlestore/backup/kubestash/logical/examples/restored-singlestore.yaml b/docs/guides/singlestore/backup/kubestash/logical/examples/restored-singlestore.yaml new file mode 100644 index 0000000000..dd4b4f93bb --- /dev/null +++ b/docs/guides/singlestore/backup/kubestash/logical/examples/restored-singlestore.yaml @@ -0,0 +1,52 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: restored-singlestore + namespace: demo +spec: + version: "8.7.10" + init: + waitForInitialRestore: true + topology: + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + storageType: Durable + deletionPolicy: WipeOut \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/logical/examples/restoresession.yaml b/docs/guides/singlestore/backup/kubestash/logical/examples/restoresession.yaml index 2438d4533a..c3a11d8cd3 100644 --- a/docs/guides/singlestore/backup/kubestash/logical/examples/restoresession.yaml +++ b/docs/guides/singlestore/backup/kubestash/logical/examples/restoresession.yaml @@ -1,21 +1,21 @@ apiVersion: core.kubestash.com/v1alpha1 kind: RestoreSession metadata: - name: restore-sample-mysql + name: restore-sample-singlestore namespace: demo spec: target: apiGroup: kubedb.com - kind: MySQL + kind: Singlestore namespace: demo - name: restored-mysql + name: restored-singlestore dataSource: - repository: gcs-mysql-repo + repository: gcs-singlestore-repo snapshot: latest encryptionSecret: name: encrypt-secret namespace: demo addon: - name: mysql-addon + name: singlestore-addon tasks: - - name: logical-backup-restore + - name: logical-backup-restore \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/logical/index.md b/docs/guides/singlestore/backup/kubestash/logical/index.md index 5ab08c5c30..a4180a739e 100644 --- a/docs/guides/singlestore/backup/kubestash/logical/index.md +++ b/docs/guides/singlestore/backup/kubestash/logical/index.md @@ -23,7 +23,7 @@ This guide will give you how you can take backup and restore your `SingleStore` - Install `KubeDB` in your cluster following the steps [here](/docs/setup/README.md). - Install `KubeStash` in your cluster following the steps [here](https://kubestash.com/docs/latest/setup/install/kubestash). - Install KubeStash `kubectl` plugin following the steps [here](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). -- If you are not familiar with how KubeStash backup and restore MySQL databases, please check the following guide [here](/docs/guides/mysql/backup/kubestash/overview/index.md). +- If you are not familiar with how KubeStash backup and restore SingleStore databases, please check the following guide [here](/docs/guides/singlestore/backup/kubestash/overview/index.md). You should be familiar with the following `KubeStash` concepts: @@ -42,11 +42,11 @@ $ kubectl create ns demo namespace/demo created ``` -> **Note:** YAML files used in this tutorial are stored in [docs/guides/mysql/backup/kubestash/logical/examples](docs/guides/mysql/backup/kubestash/logical/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. +> **Note:** YAML files used in this tutorial are stored in [docs/guides/singlestore/backup/kubestash/logical/examples](docs/guides/singlestore/backup/kubestash/logical/examples) directory of [kubedb/docs](https://github.com/kubedb/docs) repository. ## Backup SingleStore -KubeStash supports backups for `MySQL` instances across different configurations, including Standalone, and Cluster setups. In this demonstration, we'll focus on a `SingleStore` database using Clustering mode. The backup and restore process is similar for Standalone and Cluster configurations as well. +KubeStash supports backups for `SingleStore` instances across different configurations, including Standalone, and Cluster setups. In this demonstration, we'll focus on a `SingleStore` database using Clustering mode. The backup and restore process is similar for Standalone and Cluster configurations as well. This section will demonstrate how to backup a `SingleStore` database. Here, we are going to deploy a `SingleStore` database using KubeDB. Then, we are going to backup this database into a `GCS` bucket. Finally, we are going to restore the backup up data into another `SingleStore` database. @@ -146,7 +146,7 @@ KubeDB will deploy a SingleStore database according to the above specification. Let's check if the database is ready to use, ```bash -$ kubectl get mysqls.kubedb.com -n demo +$ kubectl get singlestores.kubedb.com -n demo NAME VERSION STATUS AGE sdb-sample 8.7.10 Ready 4m22s ``` @@ -269,7 +269,7 @@ kubectl get secret -n demo sdb-sample-root-cred -o jsonpath='{.data.password}'| xEJv73q3w_m1~H.G⏎ ``` -Now, Lets exec into the any aggregator `Pod` to enter into `memsql` shell and create a database and a table, +Now, Lets exec into the any aggregator `Pod` to enter into `mysql` shell and create a database and a table, ```bash $ kubectl exec -it -n demo sdb-sample-aggregator-0 -- singlestore --user=root --password=xEJv73q3w_m1~H.G @@ -378,7 +378,7 @@ spec: Let's create the BackupStorage we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/backupstorage.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/logical/examples/backupstorage.yaml backupstorage.storage.kubestash.com/gcs-storage created ``` @@ -411,7 +411,7 @@ spec: Let’s create the above `RetentionPolicy`, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/retentionpolicy.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/logical/examples/retentionpolicy.yaml retentionpolicy.storage.kubestash.com/demo-retention created ``` @@ -525,7 +525,7 @@ KubeStash triggers an instant backup as soon as the `BackupConfiguration` is rea $ kubectl get backupsession -n demo -w NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE -sample-sdb-backup-frequent-backup-1724065200 BackupConfiguration sample-mysql-backup Succeeded 7m22s +sample-sdb-backup-frequent-backup-1724065200 BackupConfiguration sdb-sample-backup Succeeded 7m22s ``` We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. @@ -618,7 +618,7 @@ status: ... ``` -> KubeStash uses the `mysqldump` command to take backups of target MySQL databases. Therefore, the component name for logical backups is set as `dump`. +> KubeStash uses the `mysqldump` command to take backups of target SingleStore databases. Therefore, the component name for logical backups is set as `dump`. Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/singlestore/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/dep/snapshots` directory. @@ -630,98 +630,130 @@ In this section, we are going to restore the database from the backup we have ta #### Deploy Restored Database: -Now, we have to deploy the restored database similarly as we have deployed the original `sample-mysql` database. However, this time there will be the following differences: +Now, we have to deploy the restored database similarly as we have deployed the original `sdb-sample` database. However, this time there will be the following differences: - We are going to specify `.spec.init.waitForInitialRestore` field that tells KubeDB to wait for first restore to complete before marking this database is ready to use. -Below is the YAML for `MySQL` CRD we are going deploy to initialize from backup, +Below is the YAML for `SingleStore` CRD we are going deploy to initialize from backup, ```yaml -apiVersion: kubedb.com/v1 -kind: MySQL +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore metadata: - name: restored-mysql + name: restored-singlestore namespace: demo spec: + version: "8.7.10" init: waitForInitialRestore: true - version: "8.2.0" - replicas: 3 topology: - mode: GroupReplication + aggregator: + replicas: 2 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + leaf: + replicas: 3 + podTemplate: + spec: + containers: + - name: singlestore + resources: + limits: + memory: "2Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "600m" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret storageType: Durable - storage: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Mi deletionPolicy: WipeOut ``` Let's create the above database, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/restored-mysql.yaml -mysql.kubedb.com/restored-mysql created +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/logical/examples/restored-singlestore.yaml +singlestore.kubedb.com/restored-singlestore created ``` If you check the database status, you will see it is stuck in `Provisioning` state. ```bash -$ kubectl get my -n demo restored-mysql -NAME VERSION STATUS AGE -restored-mysql 8.2.0 Provisioning 61s +$ kubectl get singlestore -n demo restored-singlestore +NAME VERSION STATUS AGE +restored-singlestore 8.7.10 Provisioning 61s ``` #### Create RestoreSession: -Now, we need to create a RestoreSession CRD pointing to targeted `MySQL` database. +Now, we need to create a RestoreSession CRD pointing to targeted `SingleStore` database. -Below, is the contents of YAML file of the `RestoreSession` object that we are going to create to restore backed up data into the newly created database provisioned by MySQL object named `restored-mysql`. +Below, is the contents of YAML file of the `RestoreSession` object that we are going to create to restore backed up data into the newly created database provisioned by SingleStore object named `restored-singlestore`. ```yaml apiVersion: core.kubestash.com/v1alpha1 kind: RestoreSession metadata: - name: restore-sample-mysql + name: restore-sample-singlestore namespace: demo spec: target: apiGroup: kubedb.com - kind: MySQL + kind: Singlestore namespace: demo - name: restored-mysql + name: restored-singlestore dataSource: - repository: gcs-mysql-repo + repository: gcs-singlestore-repo snapshot: latest encryptionSecret: name: encrypt-secret namespace: demo addon: - name: mysql-addon + name: singlestore-addon tasks: - name: logical-backup-restore ``` Here, -- `.spec.target` refers to the newly created `restored-mysql` MySQL object to where we want to restore backup data. +- `.spec.target` refers to the newly created `restored-singlestore` SingleStore object to where we want to restore backup data. - `.spec.dataSource.repository` specifies the Repository object that holds the backed up data. - `.spec.dataSource.snapshot` specifies to restore from latest `Snapshot`. Let's create the RestoreSession CRD object we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/logical/examples/restoresession.yaml -restoresession.core.kubestash.com/sample-mysql-restore created +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/logical/examples/restoresession.yaml +restoresession.core.kubestash.com/sample-singlestore-restore created ``` Once, you have created the `RestoreSession` object, KubeStash will create restore Job. Run the following command to watch the phase of the `RestoreSession` object, ```bash $ watch kubectl get restoresession -n demo -Every 2.0s: kubectl get restores... AppsCode-PC-03: Wed Aug 21 10:44:05 2024 +Every 2.0s: kubectl get restores... AppsCode-PC-03: Wed Sep 20 10:44:05 2024 NAME REPOSITORY FAILURE-POLICY PHASE DURATION AGE sample-restore gcs-demo-repo Succeeded 3s 53s @@ -737,40 +769,44 @@ In this section, we are going to verify whether the desired data has been restor At first, check if the database has gone into `Ready` state by the following command, ```bash -$ kubectl get my -n demo restored-mysql -NAME VERSION STATUS AGE -restored-mysql 8.2.0 Ready 34m +$ kubectl get singlestore -n demo restored-singlestore +NAME VERSION STATUS AGE +restored-singlestore 8.7.10 Ready 34m ``` Now, find out the database `Pod` by the following command, ```bash -$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=restored-mysql" -NAME READY STATUS RESTARTS AGE -restored-mysql-0 1/1 Running 0 39m +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=restored-singlestore" +NAME READY STATUS RESTARTS AGE +restored-singlestore-aggregator-0 2/2 Running 0 34m +restored-singlestore-aggregator-1 2/2 Running 0 34m +restored-singlestore-leaf-0 2/2 Running 0 34m +restored-singlestore-leaf-1 2/2 Running 0 34m +restored-singlestore-leaf-2 2/2 Running 0 34m ``` -And then copy the user name and password of the `root` user to access into `mysql` shell. +And then copy the user name and password of the `root` user to access into `memsql` shell. ```bash -$ kubectl get secret -n demo restored-mysql-auth -o jsonpath='{.data.username}'| base64 -d -root +$ kubectl get secret -n demo restored-singlestore-root-cred -o jsonpath='{.data.username}'| base64 -d +root⏎ -$ kubectl get secret -n demo restored-mysql-auth -o jsonpath='{.data.password}'| base64 -d -QMm1hi0T*7QFz_yh +kubectl get secret -n demo restored-singlestore-root-cred -o jsonpath='{.data.password}'| base64 -d +QMm1hi0T*7QFz_yh⏎ ``` -Now, let's exec into the Pod to enter into `mysql` shell and verify restored data, +Now, Lets exec into the any aggregator `Pod` to enter into `mysql` shell and create a database and a table, ```bash -$ kubectl exec -it -n demo restored-mysql-0 -- mysql --user=root --password='QMm1hi0T*7QFz_yh' -Defaulted container "mysql" out of: mysql, mysql-coordinator, mysql-init (init) -mysql: [Warning] Using a password on the command line interface can be insecure. +$ kubectl exec -it -n demo restored-singlestore-aggregator-0 -- singlestore --user=root --password=QMm1hi0T*7QFz_yh +Defaulted container "singlestore" out of: singlestore, singlestore-coordinator, singlestore-init (init) +singlestore-client: [Warning] Using a password on the command line interface can be insecure. Welcome to the MySQL monitor. Commands end with ; or \g. -Your MySQL connection id is 243 -Server version: 8.2.0 MySQL Community Server - GPL +Your MySQL connection id is 300070 +Server version: 5.7.32 SingleStoreDB source distribution (compatible; MySQL Enterprise & MySQL Commercial) -Copyright (c) 2000, 2023, Oracle and/or its affiliates. +Copyright (c) 2000, 2022, Oracle and/or its affiliates. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective @@ -778,19 +814,22 @@ owners. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. -mysql> SHOW DATABASES; +singlestore> show databases; +--------------------+ | Database | +--------------------+ +| cluster | +| demo | +| det | | information_schema | -| mysql | -| performance_schema | +| memsql | | playground | -| sys | +| singlestore_health | +| test | +--------------------+ -5 rows in set (0.00 sec) +8 rows in set (0.00 sec) -mysql> SHOW TABLES IN playground; +singlestore> SHOW TABLES IN playground; +----------------------+ | Tables_in_playground | +----------------------+ @@ -798,16 +837,17 @@ mysql> SHOW TABLES IN playground; +----------------------+ 1 row in set (0.00 sec) -mysql> SELECT * FROM playground.equipment; +singlestore> SELECT * FROM playground.equipment; +----+-------+-------+-------+ | id | type | quant | color | +----+-------+-------+-------+ | 1 | slide | 2 | blue | +----+-------+-------+-------+ -1 row in set (0.00 sec) +1 row in set (0.14 sec) -mysql> exit +singlestore> exit Bye + ``` So, from the above output, we can see that the `playground` database and the `equipment` table we have created earlier in the original database and now, they are restored successfully. @@ -817,12 +857,12 @@ So, from the above output, we can see that the `playground` database and the `eq To cleanup the Kubernetes resources created by this tutorial, run: ```bash -kubectl delete backupconfigurations.core.kubestash.com -n demo sample-mysql-backup -kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-mysql +kubectl delete backupconfigurations.core.kubestash.com -n demo sample-sdb-backup +kubectl delete restoresessions.core.kubestash.com -n demo sample-restore kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention kubectl delete backupstorage -n demo gcs-storage kubectl delete secret -n demo gcs-secret kubectl delete secret -n demo encrypt-secret -kubectl delete my -n demo restored-mysql -kubectl delete my -n demo sample-mysql +kubectl delete my -n demo restored-singlestore +kubectl delete my -n demo sdb-sample ``` \ No newline at end of file From 2fa444f12c4d7380997797b812d510e28146bda1 Mon Sep 17 00:00:00 2001 From: Ashraful Haque Tani Date: Fri, 20 Sep 2024 16:41:29 +0600 Subject: [PATCH 10/10] add application-level Signed-off-by: Ashraful Haque Tani --- .../examples/backupconfiguration.yaml | 12 +- .../examples/restoresession.yaml | 8 +- .../kubestash/application-level/index.md | 344 ++++++++++-------- 3 files changed, 201 insertions(+), 163 deletions(-) diff --git a/docs/guides/singlestore/backup/kubestash/application-level/examples/backupconfiguration.yaml b/docs/guides/singlestore/backup/kubestash/application-level/examples/backupconfiguration.yaml index 4d9e1fbae3..4f82f73efe 100644 --- a/docs/guides/singlestore/backup/kubestash/application-level/examples/backupconfiguration.yaml +++ b/docs/guides/singlestore/backup/kubestash/application-level/examples/backupconfiguration.yaml @@ -1,14 +1,14 @@ apiVersion: core.kubestash.com/v1alpha1 kind: BackupConfiguration metadata: - name: sample-mysql-backup + name: sample-singlestore-backup namespace: demo spec: target: apiGroup: kubedb.com - kind: MySQL + kind: Singlestore namespace: demo - name: sample-mysql + name: sample-singlestore backends: - name: gcs-backend storageRef: @@ -24,14 +24,14 @@ spec: jobTemplate: backoffLimit: 1 repositories: - - name: gcs-mysql-repo + - name: gcs-singlestore-repo backend: gcs-backend - directory: /mysql + directory: /singlestore encryptionSecret: name: encrypt-secret namespace: demo addon: - name: mysql-addon + name: singlestore-addon tasks: - name: manifest-backup - name: logical-backup \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/application-level/examples/restoresession.yaml b/docs/guides/singlestore/backup/kubestash/application-level/examples/restoresession.yaml index fa1fe6e950..cf266b7f6c 100644 --- a/docs/guides/singlestore/backup/kubestash/application-level/examples/restoresession.yaml +++ b/docs/guides/singlestore/backup/kubestash/application-level/examples/restoresession.yaml @@ -1,21 +1,21 @@ apiVersion: core.kubestash.com/v1alpha1 kind: RestoreSession metadata: - name: restore-sample-mysql + name: restore-sample-singlestore namespace: demo spec: manifestOptions: restoreNamespace: dev - mySQL: + singlestore: db: true dataSource: - repository: gcs-mysql-repo + repository: gcs-singlestore-repo snapshot: latest encryptionSecret: name: encrypt-secret namespace: demo addon: - name: mysql-addon + name: singlestore-addon tasks: - name: logical-backup-restore - name: manifest-restore \ No newline at end of file diff --git a/docs/guides/singlestore/backup/kubestash/application-level/index.md b/docs/guides/singlestore/backup/kubestash/application-level/index.md index a172c3eb5c..792a59700b 100644 --- a/docs/guides/singlestore/backup/kubestash/application-level/index.md +++ b/docs/guides/singlestore/backup/kubestash/application-level/index.md @@ -122,42 +122,50 @@ spec: deletionPolicy: WipeOut ``` + Here, -- `.spec.topology` specifies about the clustering configuration of MySQL. -- `.Spec.topology.mode` specifies the mode of MySQL Cluster. During the demonstration we consider to use `GroupReplication`. -Create the above `MySQL` CR, +- `spec.version` is the name of the SinglestoreVersion CRD where the docker images are specified. In this tutorial, a SingleStore `8.7.10` database is going to be created. +- `spec.topology` specifies that it will be used as cluster mode. If this field is nil it will be work as standalone mode. +- `spec.topology.aggregator.replicas` or `spec.topology.leaf.replicas` specifies that the number replicas that will be used for aggregator or leaf. +- `spec.storageType` specifies the type of storage that will be used for SingleStore database. It can be `Durable` or `Ephemeral`. Default value of this field is `Durable`. If `Ephemeral` is used then KubeDB will create SingleStore database using `EmptyDir` volume. In this case, you don't have to specify `spec.storage` field. This is useful for testing purposes. +- `spec.topology.aggregator.storage` or `spec.topology.leaf.storage` specifies the StorageClass of PVC dynamically allocated to store data for this database. This storage spec will be passed to the PetSet created by KubeDB operator to run database pods. You can specify any StorageClass available in your cluster with appropriate resource requests. +- `spec.deletionPolicy` gives flexibility whether to `nullify`(reject) the delete operation of `Singlestore` crd or which resources KubeDB should keep or delete when you delete `Singlestore` crd. If admission webhook is enabled, It prevents users from deleting the database as long as the `spec.deletionPolicy` is set to `DoNotTerminate`. Learn details of all `DeletionPolicy` [here](/docs/guides/mysql/concepts/database/index.md#specdeletionpolicy) + +> Note: `spec.storage` section is used to create PVC for database pod. It will create PVC with storage size specified in `storage.resources.requests` field. Don't specify limits here. PVC does not get resized automatically. + +Create the above `SingleStore` CR, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/sample-mysql.yaml -mysql.kubedb.com/sample-mysql created +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/application-level/examples/sample-singlestore.yaml +singlestore.kubedb.com/sample-singlestore created ``` -KubeDB will deploy a MySQL database according to the above specification. It will also create the necessary Secrets and Services to access the database. +KubeDB will deploy a SingleStore database according to the above specification. It will also create the necessary Secrets and Services to access the database. Let's check if the database is ready to use, ```bash -$ kubectl get mysqls.kubedb.com -n demo -NAME VERSION STATUS AGE -sample-mysql 8.2.0 Ready 4m22s +$ kubectl get singlestores.kubedb.com -n demo +NAME VERSION STATUS AGE +sample-singlestore 8.7.10 Ready 4m22s ``` The database is `Ready`. Verify that KubeDB has created a `Secret` and a `Service` for this database using the following commands, ```bash -$ kubectl get secret -n demo -NAME TYPE DATA AGE -sample-mysql-auth Opaque 2 4m58s +$ kubectl get secret -n demo -l=app.kubernetes.io/instance=sample-singlestore +NAME TYPE DATA AGE +sample-singlestore-root-cred kubernetes.io/basic-auth 2 4m58s + +$ kubectl get service -n demo -l=app.kubernetes.io/instance=sample-singlestore +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +sample-singlestore ClusterIP 10.128.230.168 3306/TCP,8081/TCP 5m10s +sample-singlestore-pods ClusterIP None 3306/TCP 5m10s -$ kubectl get service -n demo -l=app.kubernetes.io/instance=sample-mysql -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -sample-mysql ClusterIP 10.96.55.61 3306/TCP 97s -sample-mysql-pods ClusterIP None 3306/TCP 97s -sample-mysql-standby ClusterIP 10.96.211.186 3306/TCP 97 ``` -Here, we have to use service `sample-mysql` and secret `sample-mysql-auth` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/mysql/concepts/appbinding/index.md) CR that holds the necessary information to connect with the database. +Here, we have to use service `sample-singlestore` and secret `sample-singlestore-root-cred` to connect with the database. `KubeDB` creates an [AppBinding](/docs/guides/singlestore/concepts/appbinding/index.md) CR that holds the necessary information to connect with the database. **Verify AppBinding:** @@ -165,96 +173,113 @@ Verify that the `AppBinding` has been created successfully using the following c ```bash $ kubectl get appbindings -n demo -NAME AGE -sample-mysql 9m24s +NAME AGE +sample-singlestore 9m24s ``` Let's check the YAML of the above `AppBinding`, ```bash -$ kubectl get appbindings -n demo sample-mysql -o yaml +$ kubectl get appbindings -n demo sample-singlestore -o yaml ``` ```yaml apiVersion: appcatalog.appscode.com/v1alpha1 kind: AppBinding metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"kubedb.com/v1alpha2","kind":"Singlestore","metadata":{"annotations":{},"name":"sample-singlestore","namespace":"demo"},"spec":{"deletionPolicy":"WipeOut","licenseSecret":{"name":"license-secret"},"storageType":"Durable","topology":{"aggregator":{"podTemplate":{"spec":{"containers":[{"name":"singlestore","resources":{"limits":{"cpu":"0.6","memory":"2Gi"},"requests":{"cpu":"0.6","memory":"2Gi"}}}]}},"replicas":2,"storage":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"1Gi"}}}},"leaf":{"podTemplate":{"spec":{"containers":[{"name":"singlestore","resources":{"limits":{"cpu":"0.6","memory":"2Gi"},"requests":{"cpu":"0.6","memory":"2Gi"}}}]}},"replicas":3,"storage":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}}}},"version":"8.7.10"}} + creationTimestamp: "2024-09-11T07:03:44Z" + generation: 1 labels: app.kubernetes.io/component: database - app.kubernetes.io/instance: sample-mysql + app.kubernetes.io/instance: sample-singlestor app.kubernetes.io/managed-by: kubedb.com - app.kubernetes.io/name: mysqls.kubedb.com - name: sample-mysql + app.kubernetes.io/name: singlestores.kubedb.com + name: sample-singlestore namespace: demo ownerReferences: - - apiVersion: kubedb.com/v1 + - apiVersion: kubedb.com/v1alpha2 blockOwnerDeletion: true controller: true - kind: MySQL - name: sample-mysql - uid: edde3e8b-7775-4f91-85a9-4ba4b96315f7 - resourceVersion: "5126" - uid: 86c9a149-f8ab-44c4-947f-5f9b402aad6c + kind: Singlestore + name: sample-singlestore + uid: e08e1f37-d869-437d-9b15-14c6aef3f406 + resourceVersion: "4904220" + uid: 92b2b318-1874-4471-97ec-d789c6e16809 spec: appRef: apiGroup: kubedb.com - kind: MySQL - name: sample-mysql + kind: Singlestore + name: sample-singlestore namespace: demo clientConfig: service: - name: sample-mysql + name: sample-singlestore path: / port: 3306 scheme: tcp - url: tcp(sample-mysql.demo.svc:3306)/ - ... - ... + url: tcp(sample-singlestore.demo.svc:3306)/ + parameters: + apiVersion: config.kubedb.com/v1alpha1 + kind: SinglestoreConfiguration + masterAggregator: sample-singlestore-aggregator-0.sample-singlestore-pods.demo.svc + stash: + addon: + backupTask: + name: "" + restoreTask: + name: "" secret: - name: sample-mysql-auth - type: kubedb.com/mysql - version: 8.2.0 + name: sample-singlestore-root-cred + type: kubedb.com/singlestore + version: 8.7.10 + ``` KubeStash uses the `AppBinding` CR to connect with the target database. It requires the following two fields to set in AppBinding's `.spec` section. +- `.spec.parameters.masterAggregator` specifies the dns of master aggregator node that we have to mention in mysqldump command when taken backup or restore. - `.spec.clientConfig.service.name` specifies the name of the Service that connects to the database. - `.spec.secret` specifies the name of the Secret that holds necessary credentials to access the database. - `spec.type` specifies the types of the app that this AppBinding is pointing to. KubeDB generated AppBinding follows the following format: `/`. **Insert Sample Data:** -Now, we are going to exec into the database pod and create some sample data. At first, find out the database Pod using the following command, +Now, we are going to exec into the any aggregator pod and create some sample data. At first, find out the database `Pod` using the following command, ```bash -$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-mysql" -NAME READY STATUS RESTARTS AGE -sample-mysql-0 2/2 Running 0 33m -sample-mysql-1 2/2 Running 0 33m -sample-mysql-2 2/2 Running 0 33m +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-singlestore" +NAME READY STATUS RESTARTS AGE +sample-singlestore-aggregator-0 2/2 Running 0 15m +sample-singlestore-aggregator-1 2/2 Running 0 15m +sample-singlestore-leaf-0 2/2 Running 0 15m +sample-singlestore-leaf-1 2/2 Running 0 15m +sample-singlestore-leaf-2 2/2 Running 0 15m ``` -And copy the username and password of the `root` user to access into `mysql` shell. +And copy the username and password of the `root` user to access into `memsql` shell. ```bash -$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.username}'| base64 -d -root⏎ +$ kubectl get secret -n demo sample-singlestore-root-cred -o jsonpath='{.data.username}'| base64 -d +root⏎ -$ kubectl get secret -n demo sample-mysql-auth -o jsonpath='{.data.password}'| base64 -d -DZfmUZd14fNEEOU4⏎ +kubectl get secret -n demo sample-singlestore-root-cred -o jsonpath='{.data.password}'| base64 -d +xEJv73q3w_m1~H.G⏎ ``` -Now, Lets exec into the Pod to enter into `mysql` shell and create a database and a table, +Now, Lets exec into the any aggregator `Pod` to enter into `mysql` shell and create a database and a table, ```bash -$ kubectl exec -it -n demo sample-mysql-0 -- mysql --user=root --password=DZfmUZd14fNEEOU4 -Defaulted container "mysql" out of: mysql, mysql-init (init) -mysql: [Warning] Using a password on the command line interface can be insecure. +$ kubectl exec -it -n demo sample-singlestore-aggregator-0 -- singlestore --user=root --password=xEJv73q3w_m1~H.G +Defaulted container "singlestore" out of: singlestore, singlestore-coordinator, singlestore-init (init) +singlestore-client: [Warning] Using a password on the command line interface can be insecure. Welcome to the MySQL monitor. Commands end with ; or \g. -Your MySQL connection id is 977 -Server version: 8.2.0 MySQL Community Server - GPL +Your MySQL connection id is 300070 +Server version: 5.7.32 SingleStoreDB source distribution (compatible; MySQL Enterprise & MySQL Commercial) -Copyright (c) 2000, 2023, Oracle and/or its affiliates. +Copyright (c) 2000, 2022, Oracle and/or its affiliates. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective @@ -262,46 +287,51 @@ owners. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. -mysql> CREATE DATABASE playground; +singlestore> CREATE DATABASE playground partitions 2; Query OK, 1 row affected (0.01 sec) -mysql> SHOW DATABASES; +singlestore> show databases; +--------------------+ | Database | +--------------------+ +| cluster | +| demo | +| det | | information_schema | -| mysql | -| performance_schema | +| memsql | | playground | -| sys | +| singlestore_health | +| test | +--------------------+ -5 rows in set (0.00 sec) +8 rows in set (0.00 sec) -mysql> CREATE TABLE playground.equipment ( id INT NOT NULL AUTO_INCREMENT, type VARCHAR(50), quant INT, color VARCHAR(25), PRIMARY KEY(id)); -Query OK, 0 rows affected (0.01 sec) +singlestore> CREATE TABLE playground.equipment ( id INT NOT NULL AUTO_INCREMENT, type VARCHAR(50), quant INT, color VARCHAR(25), PRIMARY KEY(id)); +Query OK, 0 rows affected, 1 warning (0.27 sec) -mysql> SHOW TABLES IN playground; +singlestore> SHOW TABLES IN playground; +----------------------+ | Tables_in_playground | +----------------------+ | equipment | +----------------------+ -1 row in set (0.01 sec) +1 row in set (0.00 sec) -mysql> INSERT INTO playground.equipment (type, quant, color) VALUES ("slide", 2, "blue"); -Query OK, 1 row affected (0.01 sec) +singlestore> INSERT INTO playground.equipment (type, quant, color) VALUES ("slide", 2, "blue"); +Query OK, 1 row affected (1.15 sec) -mysql> SELECT * FROM playground.equipment; +singlestore> SELECT * FROM playground.equipment; +----+-------+-------+-------+ | id | type | quant | color | +----+-------+-------+-------+ | 1 | slide | 2 | blue | +----+-------+-------+-------+ -1 row in set (0.00 sec) +1 row in set (0.14 sec) -mysql> exit +singlestore> exit Bye + ``` + Now, we are ready to backup the database. ### Prepare Backend @@ -348,7 +378,7 @@ spec: Let's create the BackupStorage we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/backupstorage.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/application-level/examples/backupstorage.yaml backupstorage.storage.kubestash.com/gcs-storage created ``` @@ -381,13 +411,13 @@ spec: Let’s create the above `RetentionPolicy`, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/retentionpolicy.yaml +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/application-level/examples/retentionpolicy.yaml retentionpolicy.storage.kubestash.com/demo-retention created ``` ### Backup -We have to create a `BackupConfiguration` targeting respective `sample-mysql` MySQL database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. +We have to create a `BackupConfiguration` targeting respective `sample-singlestore` SingleStore database. Then, KubeStash will create a `CronJob` for each session to take periodic backup of that database. At first, we need to create a secret with a Restic password for backup data encryption. @@ -404,20 +434,20 @@ secret "encrypt-secret" created **Create BackupConfiguration:** -Below is the YAML for `BackupConfiguration` CR to take application-level backup of the `sample-mysql` database that we have deployed earlier, +Below is the YAML for `BackupConfiguration` CR to take application-level backup of the `sample-singlestore` database that we have deployed earlier, ```yaml apiVersion: core.kubestash.com/v1alpha1 kind: BackupConfiguration metadata: - name: sample-mysql-backup + name: sample-singlestore-backup namespace: demo spec: target: apiGroup: kubedb.com - kind: MySQL + kind: Singlestore namespace: demo - name: sample-mysql + name: sample-singlestore backends: - name: gcs-backend storageRef: @@ -433,28 +463,28 @@ spec: jobTemplate: backoffLimit: 1 repositories: - - name: gcs-mysql-repo + - name: gcs-singlestore-repo backend: gcs-backend - directory: /mysql + directory: /singlestore encryptionSecret: name: encrypt-secret namespace: demo addon: - name: mysql-addon + name: singlestore-addon tasks: - name: manifest-backup - name: logical-backup ``` - `.spec.sessions[*].schedule` specifies that we want to backup at `5 minutes` interval. -- `.spec.target` refers to the targeted `sample-mysql` MySQL database that we created earlier. +- `.spec.target` refers to the targeted `sample-singlestore` SingleStore database that we created earlier. - `.spec.sessions[*].addon.tasks[*].name[*]` specifies that both the `manifest-backup` and `logical-backup` tasks will be executed. Let's create the `BackupConfiguration` CR that we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/backupconfiguration.yaml -backupconfiguration.core.kubestash.com/sample-mysql-backup created +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/application-level/examples/backupconfiguration.yaml +backupconfiguration.core.kubestash.com/sample-singlestore-backup created ``` **Verify Backup Setup Successful** @@ -464,7 +494,7 @@ If everything goes well, the phase of the `BackupConfiguration` should be `Ready ```bash $ kubectl get backupconfiguration -n demo NAME PHASE PAUSED AGE -sample-mysql-backup Ready 2m50s +sample-singlestore-backup Ready 2m50s ``` Additionally, we can verify that the `Repository` specified in the `BackupConfiguration` has been created using the following command, @@ -472,10 +502,10 @@ Additionally, we can verify that the `Repository` specified in the `BackupConfig ```bash $ kubectl get repo -n demo NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE -gcs-mysql-repo 0 0 B Ready 3m +gcs-singlestore-repo 0 0 B Ready 3m ``` -KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/mysql` directory. +KubeStash keeps the backup for `Repository` YAMLs. If we navigate to the GCS bucket, we will see the `Repository` YAML stored in the `demo/singlestore` directory. **Verify CronJob:** @@ -486,7 +516,7 @@ Verify that the `CronJob` has been created using the following command, ```bash $ kubectl get cronjob -n demo NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE -trigger-sample-mysql-backup-frequent-backup */5 * * * * 0 2m45s 3m25s +trigger-sample-singlestore-backup-frequent-backup */5 * * * * 0 2m45s 3m25s ``` **Verify BackupSession:** @@ -499,19 +529,19 @@ Run the following command to watch `BackupSession` CR, $ kubectl get backupsession -n demo -w NAME INVOKER-TYPE INVOKER-NAME PHASE DURATION AGE -sample-mysql-backup-frequent-backup-1724065200 BackupConfiguration sample-mysql-backup Succeeded 7m22s +sample-singlestore-backup-frequent-backup-1724065200 BackupConfiguration sample-singlestore-backup Succeeded 7m22s ``` We can see from the above output that the backup session has succeeded. Now, we are going to verify whether the backed up data has been stored in the backend. **Verify Backup:** -Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-mysql-backup` has been updated by the following command, +Once a backup is complete, KubeStash will update the respective `Repository` CR to reflect the backup. Check that the repository `sample-singlestore-backup` has been updated by the following command, ```bash -$ kubectl get repository -n demo gcs-mysql-repo +$ kubectl get repository -n demo gcs-singlestore-repo NAME INTEGRITY SNAPSHOT-COUNT SIZE PHASE LAST-SUCCESSFUL-BACKUP AGE -gcs-mysql-repo true 1 806 B Ready 8m27s 9m18s +gcs-singlestore-repo true 1 806 B Ready 8m27s 9m18s ``` At this moment we have one `Snapshot`. Run the following command to check the respective `Snapshot` which represents the state of a backup run for an application. @@ -519,7 +549,7 @@ At this moment we have one `Snapshot`. Run the following command to check the re ```bash $ kubectl get snapshots -n demo -l=kubestash.com/repo-name=gcs-demo-repo NAME REPOSITORY SESSION SNAPSHOT-TIME DELETION-POLICY PHASE AGE -gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 sample-mysql-backup frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h +gcs-singlestore-repo-sample-singlestore-backup-frequent-backup-1725359100 sample-singlestore-backup frequent-backup 2024-01-23T13:10:54Z Delete Succeeded 16h ``` > Note: KubeStash creates a `Snapshot` with the following labels: @@ -533,7 +563,7 @@ gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 sample-mysql-bac If we check the YAML of the `Snapshot`, we can find the information about the backed up components of the Database. ```bash -$ kubectl get snapshots -n demo gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 -oyaml +$ kubectl get snapshots -n demo gcs-singlestore-repo-sample-singlestore-backup-frequent-backup-1725359100 -oyaml ``` ```yaml @@ -545,32 +575,32 @@ metadata: - kubestash.com/cleanup generation: 1 labels: - kubestash.com/app-ref-kind: MySQL - kubestash.com/app-ref-name: sample-mysql + kubestash.com/app-ref-kind: Singlestore + kubestash.com/app-ref-name: sample-singlestore kubestash.com/app-ref-namespace: demo - kubestash.com/repo-name: gcs-mysql-repo + kubestash.com/repo-name: gcs-singlestore-repo annotations: - kubedb.com/db-version: 8.2.0 - name: gcs-mysql-repo-sample-mysql-backup-frequent-backup-1725359100 + kubedb.com/db-version: 8.7.10 + name: gcs-singlestore-repo-sample-singlestore-backup-frequent-backup-1725359100 namespace: demo ownerReferences: - apiVersion: storage.kubestash.com/v1alpha1 blockOwnerDeletion: true controller: true kind: Repository - name: gcs-mysql-repo + name: gcs-singlestore-repo uid: 1f5ba355-7f99-4b99-8bbf-9f9d4f31c52a resourceVersion: "213010" uid: 18cabb10-e594-4655-8763-3daa0872508e spec: appRef: apiGroup: kubedb.com - kind: MySQL - name: sample-mysql + kind: Singlestore + name: sample-singlestore namespace: demo - backupSession: sample-mysql-backup-frequent-backup-1725359100 + backupSession: sample-singlestore-backup-frequent-backup-1725359100 deletionPolicy: Delete - repository: gcs-mysql-repo + repository: gcs-singlestore-repo session: frequent-backup snapshotID: 01J6VPN4TPHDFT1M9Q9YVGMTKF type: FullBackup @@ -619,10 +649,10 @@ status: totalComponents: 2 ``` -> KubeStash uses the `mysqldump` command to take backups of target MySQL databases. Therefore, the component name for logical backups is set as `dump`. -> KubeStash set component name as `manifest` for the `manifest backup` of MySQL databases. +> KubeStash uses the `mysqldump` command to take backups of target SingleStore databases. Therefore, the component name for logical backups is set as `dump`. +> KubeStash set component name as `manifest` for the `manifest backup` of SingleStore databases. -Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/mysql/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/dep/snapshots` directory. +Now, if we navigate to the GCS bucket, we will see the backed up data stored in the `demo/singlestore/repository/v1/frequent-backup/dump` directory. KubeStash also keeps the backup for `Snapshot` YAMLs, which can be found in the `demo/dep/snapshots` directory. > Note: KubeStash stores all dumped data encrypted in the backup directory, meaning it remains unreadable until decrypted. @@ -649,21 +679,21 @@ Below, is the contents of YAML file of the `RestoreSession` CR that we are going apiVersion: core.kubestash.com/v1alpha1 kind: RestoreSession metadata: - name: restore-sample-mysql + name: restore-sample-singlestore namespace: demo spec: manifestOptions: restoreNamespace: dev - mySQL: + singlestore: db: true dataSource: - repository: gcs-mysql-repo + repository: gcs-singlestore-repo snapshot: latest encryptionSecret: name: encrypt-secret namespace: demo addon: - name: mysql-addon + name: singlestore-addon tasks: - name: logical-backup-restore - name: manifest-restore @@ -671,7 +701,7 @@ spec: Here, -- `.spec.manifestOptions.mySQL.db` specifies whether to restore the DB manifest or not. +- `.spec.manifestOptions.singlestore.db` specifies whether to restore the DB manifest or not. - `.spec.dataSource.repository` specifies the Repository object that holds the backed up data. - `.spec.dataSource.snapshot` specifies to restore from latest `Snapshot`. - `.spec.addon.tasks[*]` specifies that both the `manifest-restore` and `logical-backup-restore` tasks. @@ -679,8 +709,8 @@ Here, Let's create the RestoreSession CRD object we have shown above, ```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/mysql/backup/kubestash/application-level/examples/restoresession.yaml -restoresession.core.kubestash.com/sample-mysql-restore created +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/singlestore/backup/kubestash/application-level/examples/restoresession.yaml +restoresession.core.kubestash.com/sample-singlestore-restore created ``` Once, you have created the `RestoreSession` object, KubeStash will create restore Job. Run the following command to watch the phase of the `RestoreSession` object, @@ -694,17 +724,17 @@ sample-restore gcs-demo-repo Succeeded 3s 53s ``` The `Succeeded` phase means that the restore process has been completed successfully. -#### Verify Restored MySQL Manifest: +#### Verify Restored SingleStore Manifest: -In this section, we will verify whether the desired `MySQL` database manifest has been successfully applied to the cluster. +In this section, we will verify whether the desired `SingleStore` database manifest has been successfully applied to the cluster. ```bash -$ kubectl get mysqls.kubedb.com -n dev -NAME VERSION STATUS AGE -sample-mysql 8.2.0 Ready 39m +$ kubectl get singlestores.kubedb.com -n dev +NAME VERSION STATUS AGE +sample-singlestore 8.7.10 Ready 39m ``` -The output confirms that the `MySQL` database has been successfully created with the same configuration as it had at the time of backup. +The output confirms that the `SingleStore` database has been successfully created with the same configuration as it had at the time of backup. #### Verify Restored Data: @@ -713,40 +743,44 @@ In this section, we are going to verify whether the desired data has been restor At first, check if the database has gone into `Ready` state by the following command, ```bash -$ kubectl get my -n dev sample-mysql -NAME VERSION STATUS AGE -sample-mysql 8.2.0 Ready 4m +$ kubectl get sdb -n dev sample-singlestore +NAME VERSION STATUS AGE +sample-singlestore 8.7.10 Ready 4m ``` Now, find out the database `Pod` by the following command, ```bash -$ kubectl get pods -n dev --selector="app.kubernetes.io/instance=sample-mysql" -NAME READY STATUS RESTARTS AGE -sample-mysql-0 2/2 Running 0 2m -sample-mysql-1 2/2 Running 0 2m -sample-mysql-2 2/2 Running 0 2m +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=sample-singlestore" +NAME READY STATUS RESTARTS AGE +sample-singlestore-aggregator-0 2/2 Running 0 15m +sample-singlestore-aggregator-1 2/2 Running 0 15m +sample-singlestore-leaf-0 2/2 Running 0 15m +sample-singlestore-leaf-1 2/2 Running 0 15m +sample-singlestore-leaf-2 2/2 Running 0 15m ``` -And then copy the username and password of the `root` user to access into `mysql` shell. +And copy the username and password of the `root` user to access into `mysql` shell. ```bash -$ kubectl get secret -n dev sample-mysql-auth -o jsonpath='{.data.username}'| base64 -d -root +$ kubectl get secret -n demo sample-singlestore-root-cred -o jsonpath='{.data.username}'| base64 -d +root⏎ -$ kubectl get secret -n dev sample-mysql-auth -o jsonpath='{.data.password}'| base64 -d -QMm1hi0T*7QFz_yh +kubectl get secret -n demo sample-singlestore-root-cred -o jsonpath='{.data.password}'| base64 -d +xEJv73q3w_m1~H.G⏎ ``` +Now, Lets exec into the any aggregator `Pod` to enter into `mysql` shell and create a database and a table, + ```bash -$ kubectl exec -it -n dev sample-mysql-0 -- mysql --user=root --password='QMm1hi0T*7QFz_yh' -Defaulted container "mysql" out of: mysql, mysql-coordinator, mysql-init (init) -mysql: [Warning] Using a password on the command line interface can be insecure. +$ kubectl exec -it -n demo sample-singlestore-aggregator-0 -- singlestore --user=root --password=xEJv73q3w_m1~H.G +Defaulted container "singlestore" out of: singlestore, singlestore-coordinator, singlestore-init (init) +singlestore-client: [Warning] Using a password on the command line interface can be insecure. Welcome to the MySQL monitor. Commands end with ; or \g. -Your MySQL connection id is 243 -Server version: 8.2.0 MySQL Community Server - GPL +Your MySQL connection id is 300070 +Server version: 5.7.32 SingleStoreDB source distribution (compatible; MySQL Enterprise & MySQL Commercial) -Copyright (c) 2000, 2023, Oracle and/or its affiliates. +Copyright (c) 2000, 2022, Oracle and/or its affiliates. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective @@ -754,19 +788,22 @@ owners. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. -mysql> SHOW DATABASES; +singlestore> show databases; +--------------------+ | Database | +--------------------+ +| cluster | +| demo | +| det | | information_schema | -| mysql | -| performance_schema | +| memsql | | playground | -| sys | +| singlestore_health | +| test | +--------------------+ -5 rows in set (0.00 sec) +8 rows in set (0.00 sec) -mysql> SHOW TABLES IN playground; +singlestore> SHOW TABLES IN playground; +----------------------+ | Tables_in_playground | +----------------------+ @@ -774,16 +811,17 @@ mysql> SHOW TABLES IN playground; +----------------------+ 1 row in set (0.00 sec) -mysql> SELECT * FROM playground.equipment; +singlestore> SELECT * FROM playground.equipment; +----+-------+-------+-------+ | id | type | quant | color | +----+-------+-------+-------+ | 1 | slide | 2 | blue | +----+-------+-------+-------+ -1 row in set (0.00 sec) +1 row in set (0.14 sec) -mysql> exit +singlestore> exit Bye + ``` So, from the above output, we can see that the `playground` database and the `equipment` table we have created earlier in the original database and now, they are restored successfully. @@ -793,12 +831,12 @@ So, from the above output, we can see that the `playground` database and the `eq To cleanup the Kubernetes resources created by this tutorial, run: ```bash -kubectl delete backupconfigurations.core.kubestash.com -n demo sample-mysql-backup +kubectl delete backupconfigurations.core.kubestash.com -n demo sample-singlestore-backup kubectl delete backupstorage -n demo gcs-storage kubectl delete secret -n demo gcs-secret kubectl delete secret -n demo encrypt-secret kubectl delete retentionpolicies.storage.kubestash.com -n demo demo-retention -kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-mysql -kubectl delete my -n demo sample-mysql -kubectl delete my -n dev sample-mysql +kubectl delete restoresessions.core.kubestash.com -n demo restore-sample-singlestore +kubectl delete my -n demo sample-singlestore +kubectl delete my -n dev sample-singlestore ``` \ No newline at end of file