[DOCS] Add cloud store backend sync recipe (#11986)

docs/docusaurus/docs/components/examples_under_test.py

@@ -544,6 +544,33 @@
     ),
 ]
 
+docs_examples_cloud_store_backend_sync = [
+    IntegrationTestFixture(
+        # To test, run:
+        # pytest --docs-tests --bigquery -k "docs_example_cloud_store_backend_gcs" tests/integration/test_script_runner.py
+        name="docs_example_cloud_store_backend_gcs",
+        user_flow_script="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_gcs.py",
+        data_dir="docs/docusaurus/docs/components/_testing/test_data_sets/single_test_file",
+        backend_dependencies=[BackendDependencies.GCS],
+    ),
+    IntegrationTestFixture(
+        # To test, run:
+        # pytest --docs-tests --aws -k "docs_example_cloud_store_backend_aws" tests/integration/test_script_runner.py
+        name="docs_example_cloud_store_backend_aws",
+        user_flow_script="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_aws.py",
+        data_dir="docs/docusaurus/docs/components/_testing/test_data_sets/single_test_file",
+        backend_dependencies=[BackendDependencies.AWS],
+    ),
+    IntegrationTestFixture(
+        # To test, run:
+        # pytest --docs-tests --azure -k "docs_example_cloud_store_backend_azure" tests/integration/test_script_runner.py
+        name="docs_example_cloud_store_backend_azure",
+        user_flow_script="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_azure.py",
+        data_dir="docs/docusaurus/docs/components/_testing/test_data_sets/single_test_file",
+        backend_dependencies=[BackendDependencies.AZURE],
+    ),
+]
+
 learn_data_quality_use_cases = [
     # Distribution.
     IntegrationTestFixture(
@@ -677,4 +704,5 @@
 docs_tests.extend(docs_examples_trigger_actions_based_on_validation_results)
 docs_tests.extend(docs_example_configure_project_settings)
 docs_tests.extend(docs_examples_configure_data_docs)
+docs_tests.extend(docs_examples_cloud_store_backend_sync)
 docs_tests.extend(learn_data_quality_use_cases)

docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_backends/_aws.md

@@ -0,0 +1,33 @@
+## Amazon S3
+
+**Prerequisites:**
+- An S3 bucket (e.g., `s3://my-gx-bucket/`)
+- AWS CLI installed and configured (`aws configure`)
+- The `boto3` Python package (optional, for programmatic access)
+
+**Step 1: Pull config from S3**
+
+```bash
+aws s3 cp s3://my-gx-bucket/gx/ ./gx/ --recursive
+```
+
+**Step 2: Run your validation locally**
+
+```python title="Python" name="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_aws.py - run validation locally"
+```
+
+**Step 3: Push results back to S3**
+
+Use a unique key (e.g., suite name + timestamp) to avoid collisions:
+
+```bash
+RUN_KEY="my_suite_$(date +%Y%m%d_%H%M%S)"
+aws s3 cp gx/uncommitted/validations/my_suite/ s3://my-gx-bucket/gx/uncommitted/validations/my_suite/$RUN_KEY/ --recursive
+```
+
+**Step 4: Optional — Build and upload Data Docs**
+
+```bash
+python -c "import great_expectations as gx; context = gx.get_context(mode='file'); context.build_data_docs()"
+aws s3 cp gx/uncommitted/data_docs/local_site/ s3://my-gx-bucket/gx/uncommitted/data_docs/local_site/ --recursive
+```

docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_backends/_azure.md

@@ -0,0 +1,45 @@
+## Azure Blob Storage
+
+**Prerequisites:**
+- An Azure Blob container (e.g., `https://myaccount.blob.core.windows.net/my-gx-container/`)
+- Azure CLI installed and authenticated (`az login`)
+- The `azure-storage-blob` Python package (optional, for programmatic access)
+
+**Step 1: Pull config from Azure Blob**
+
+```bash
+az storage blob download-batch \
+  --account-name myaccount \
+  --source my-gx-container \
+  --pattern gx/* \
+  --destination ./
+```
+
+**Step 2: Run your validation locally**
+
+```python title="Python" name="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_azure.py - run validation locally"
+```
+
+**Step 3: Push results back to Azure Blob**
+
+Use a unique key (e.g., suite name + timestamp) to avoid collisions:
+
+```bash
+RUN_KEY="my_suite_$(date +%Y%m%d_%H%M%S)"
+az storage blob upload-batch \
+  --account-name myaccount \
+  --destination my-gx-container \
+  --source gx/uncommitted/validations/my_suite/ \
+  --destination-path gx/uncommitted/validations/my_suite/$RUN_KEY/
+```
+
+**Step 4: Optional — Build and upload Data Docs**
+
+```bash
+python -c "import great_expectations as gx; context = gx.get_context(mode='file'); context.build_data_docs()"
+az storage blob upload-batch \
+  --account-name myaccount \
+  --destination my-gx-container \
+  --source gx/uncommitted/data_docs/local_site/ \
+  --destination-path gx/uncommitted/data_docs/local_site/
+```

docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_backends/_gcs.md

@@ -0,0 +1,33 @@
+## Google Cloud Storage (GCS)
+
+**Prerequisites:**
+- A GCS bucket (e.g., `gs://my-gx-bucket/`)
+- `gsutil` CLI installed and authenticated (`gcloud auth login`)
+- The `google-cloud-storage` Python package (optional, for programmatic access)
+
+**Step 1: Pull config from GCS**
+
+```bash
+gsutil -m cp -r gs://my-gx-bucket/gx/ ./gx/
+```
+
+**Step 2: Run your validation locally**
+
+```python title="Python" name="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_gcs.py - run validation locally"
+```
+
+**Step 3: Push results back to GCS**
+
+Use a unique key (e.g., suite name + timestamp) to avoid collisions:
+
+```bash
+RUN_KEY="my_suite_$(date +%Y%m%d_%H%M%S)"
+gsutil -m cp -r gx/uncommitted/validations/my_suite/ gs://my-gx-bucket/gx/uncommitted/validations/my_suite/$RUN_KEY/
+```
+
+**Step 4: Optional — Build and upload Data Docs**
+
+```bash
+python -c "import great_expectations as gx; context = gx.get_context(mode='file'); context.build_data_docs()"
+gsutil -m cp -r gx/uncommitted/data_docs/local_site/ gs://my-gx-bucket/gx/uncommitted/data_docs/local_site/
+```

docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_aws.py

@@ -0,0 +1,68 @@
+"""
+Example: Sync-based workflow for storing GX artifacts in Amazon S3.
+
+To test, run:
+pytest --docs-tests -k "docs_example_cloud_store_backend_aws" tests/integration/test_script_runner.py
+"""
+
+import uuid
+
+
+def set_up_context_for_example(context):
+    """Create a minimal GX setup for the example."""
+    batch_definition = (
+        context.data_sources.add_pandas_filesystem(
+            name="my_data_source", base_directory="./data/folder_with_data"
+        )
+        .add_csv_asset(name="my_data_asset")
+        .add_batch_definition_path(
+            name="my_batch_definition", path="yellow_tripdata_sample_2019-01.csv"
+        )
+    )
+
+    expectation_suite = context.suites.add(
+        gx.ExpectationSuite(name="my_expectation_suite")
+    )
+    expectation_suite.add_expectation(
+        gx.expectations.ExpectColumnValuesToNotBeNull(column="pickup_datetime")
+    )
+
+    context.validation_definitions.add(
+        gx.ValidationDefinition(
+            data=batch_definition,
+            suite=expectation_suite,
+            name="my_validation_definition",
+        )
+    )
+
+
+# EXAMPLE SCRIPT STARTS HERE:
+# <snippet name="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_aws.py - full code example">
+import great_expectations as gx
+
+context = gx.get_context(mode="file")
+# Hide this
+set_up_context_for_example(context)
+
+# Step 1: Pull config from S3 (simulated)
+# In production: aws s3 cp s3://my-gx-bucket/gx/ ./gx/ --recursive
+BUCKET_NAME = "my-gx-bucket"
+SUITE_NAME = "my_expectation_suite"
+
+# Step 2: Run validation locally
+# <snippet name="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_aws.py - run validation locally">
+validation_definition = context.validation_definitions.get("my_validation_definition")
+run_name = f"{SUITE_NAME}_{uuid.uuid4().hex[:8]}"
+result = validation_definition.run(run_name=run_name)
+# </snippet>
+
+print(f"Validation {'succeeded' if result.success else 'failed'}")
+
+# Step 3: Push results back to S3 with unique key
+# In production:
+# aws s3 cp gx/uncommitted/validations/{SUITE_NAME}/{run_name}/ s3://{BUCKET_NAME}/gx/uncommitted/validations/{SUITE_NAME}/{run_name}/ --recursive
+
+# Step 4: Optional - Build and upload Data Docs
+# context.build_data_docs()
+# aws s3 cp gx/uncommitted/data_docs/local_site/ s3://{BUCKET_NAME}/gx/uncommitted/data_docs/local_site/ --recursive
+# </snippet>

docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_azure.py

@@ -0,0 +1,69 @@
+"""
+Example: Sync-based workflow for storing GX artifacts in Azure Blob Storage.
+
+To test, run:
+pytest --docs-tests -k "docs_example_cloud_store_backend_azure" tests/integration/test_script_runner.py
+"""
+
+import uuid
+
+
+def set_up_context_for_example(context):
+    """Create a minimal GX setup for the example."""
+    batch_definition = (
+        context.data_sources.add_pandas_filesystem(
+            name="my_data_source", base_directory="./data/folder_with_data"
+        )
+        .add_csv_asset(name="my_data_asset")
+        .add_batch_definition_path(
+            name="my_batch_definition", path="yellow_tripdata_sample_2019-01.csv"
+        )
+    )
+
+    expectation_suite = context.suites.add(
+        gx.ExpectationSuite(name="my_expectation_suite")
+    )
+    expectation_suite.add_expectation(
+        gx.expectations.ExpectColumnValuesToNotBeNull(column="pickup_datetime")
+    )
+
+    context.validation_definitions.add(
+        gx.ValidationDefinition(
+            data=batch_definition,
+            suite=expectation_suite,
+            name="my_validation_definition",
+        )
+    )
+
+
+# EXAMPLE SCRIPT STARTS HERE:
+# <snippet name="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_azure.py - full code example">
+import great_expectations as gx
+
+context = gx.get_context(mode="file")
+# Hide this
+set_up_context_for_example(context)
+
+# Step 1: Pull config from Azure Blob (simulated)
+# In production: az storage blob download-batch --account-name myaccount --source my-gx-container --pattern gx/* --destination ./
+ACCOUNT_NAME = "myaccount"
+CONTAINER_NAME = "my-gx-container"
+SUITE_NAME = "my_expectation_suite"
+
+# Step 2: Run validation locally
+# <snippet name="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_azure.py - run validation locally">
+validation_definition = context.validation_definitions.get("my_validation_definition")
+run_name = f"{SUITE_NAME}_{uuid.uuid4().hex[:8]}"
+result = validation_definition.run(run_name=run_name)
+# </snippet>
+
+print(f"Validation {'succeeded' if result.success else 'failed'}")
+
+# Step 3: Push results back to Azure Blob with unique key
+# In production:
+# az storage blob upload-batch --account-name {ACCOUNT_NAME} --destination {CONTAINER_NAME} --source gx/uncommitted/validations/{SUITE_NAME}/{run_name}/ --destination-path gx/uncommitted/validations/{SUITE_NAME}/{run_name}/
+
+# Step 4: Optional - Build and upload Data Docs
+# context.build_data_docs()
+# az storage blob upload-batch --account-name {ACCOUNT_NAME} --destination {CONTAINER_NAME} --source gx/uncommitted/data_docs/local_site/ --destination-path gx/uncommitted/data_docs/local_site/
+# </snippet>

docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_gcs.py

@@ -0,0 +1,72 @@
+"""
+Example: Sync-based workflow for storing GX artifacts in Google Cloud Storage.
+
+To test, run:
+pytest --docs-tests -k "docs_example_cloud_store_backend_gcs" tests/integration/test_script_runner.py
+"""
+
+import uuid
+
+
+def set_up_context_for_example(context):
+    """Create a minimal GX setup for the example."""
+    # Create a Batch Definition
+    batch_definition = (
+        context.data_sources.add_pandas_filesystem(
+            name="my_data_source", base_directory="./data/folder_with_data"
+        )
+        .add_csv_asset(name="my_data_asset")
+        .add_batch_definition_path(
+            name="my_batch_definition", path="yellow_tripdata_sample_2019-01.csv"
+        )
+    )
+
+    # Create an Expectation Suite
+    expectation_suite = context.suites.add(
+        gx.ExpectationSuite(name="my_expectation_suite")
+    )
+    expectation_suite.add_expectation(
+        gx.expectations.ExpectColumnValuesToNotBeNull(column="pickup_datetime")
+    )
+
+    # Create a Validation Definition
+    context.validation_definitions.add(
+        gx.ValidationDefinition(
+            data=batch_definition,
+            suite=expectation_suite,
+            name="my_validation_definition",
+        )
+    )
+
+
+# EXAMPLE SCRIPT STARTS HERE:
+# <snippet name="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_gcs.py - full code example">
+import great_expectations as gx
+
+context = gx.get_context(mode="file")
+# Hide this
+set_up_context_for_example(context)
+
+# Step 1: Pull config from GCS (simulated for this example)
+# In production, you would run: gsutil -m cp -r gs://my-gx-bucket/gx/ ./gx/
+BUCKET_NAME = "my-gx-bucket"
+SUITE_NAME = "my_expectation_suite"
+
+# Step 2: Run validation locally
+# <snippet name="docs/docusaurus/docs/core/configure_project_settings/cloud_store_backend_sync/_examples/cloud_store_backend_gcs.py - run validation locally">
+validation_definition = context.validation_definitions.get("my_validation_definition")
+run_name = f"{SUITE_NAME}_{uuid.uuid4().hex[:8]}"
+result = validation_definition.run(run_name=run_name)
+# </snippet>
+
+print(f"Validation {'succeeded' if result.success else 'failed'}")
+print(f"Run name: {run_name}")
+
+# Step 3: Push results back to GCS with unique key
+# In production, you would run:
+# gsutil -m cp -r gx/uncommitted/validations/{SUITE_NAME}/{run_name}/ gs://{BUCKET_NAME}/gx/uncommitted/validations/{SUITE_NAME}/{run_name}/
+
+# Step 4: Optional - Build and upload Data Docs
+# context.build_data_docs()
+# gsutil -m cp -r gx/uncommitted/data_docs/local_site/ gs://{BUCKET_NAME}/gx/uncommitted/data_docs/local_site/
+# </snippet>