[MAINTENANCE] Restore the GCS datasource tests and run them in CI

.github/workflows/ci.yml

@@ -535,6 +535,7 @@ jobs:
           - bigquery
           - databricks
           - filesystem
+          - gcs_deps
           - sql_server
           - mysql
           - postgresql

pyproject.toml

@@ -692,6 +692,7 @@ markers = [
     "docs: mark a test as a docs test.",
     "external_sqldialect: mark test as requiring install of an external sql dialect.",
     "filesystem: mark tests using the filesystem as the storage backend.",
+    "gcs_deps: mark tests that need google cloud storage dependencies but no credentials",
     "integration: mark test as an integration test.",
     "sql_server: mark a test as SQL Server-dependent.",
     "mysql: mark a test as mysql-dependent.",

reqs/requirements-dev-gcs.txt

@@ -0,0 +1,3 @@
+# needed to prevent excessive pip backtracking on python 3.11
+google-cloud-storage>=2.10.0;python_version >= '3.11'
+google-cloud-storage>=1.28.0;python_version < '3.11'

tasks.py

@@ -879,6 +879,7 @@ class TestDependencies(NamedTuple):
         services=("spark",),
         extra_pytest_args=("--spark", "--docs-tests"),
     ),
+    "gcs_deps": TestDependencies(("reqs/requirements-dev-gcs.txt",)),
     "gx-redshift": TestDependencies(
         requirement_files=("reqs/requirements-dev-gx-redshift.txt",),
     ),

tests/conftest.py

@@ -121,6 +121,7 @@
     "databricks",
     "docs",
     "filesystem",
+    "gcs_deps",
     "generic_sql",
     "integration",
     "mysql",

tests/datasource/fluent/test_pandas_google_cloud_storage_datasource.py

@@ -81,7 +81,7 @@ def object_keys() -> List[str]:
     ]
 
 
-@pytest.mark.unit
+@pytest.mark.gcs_deps
 def test_construct_pandas_gcs_datasource_without_gcs_options():
     google_cred_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
     if not google_cred_file:
@@ -97,9 +97,9 @@ def test_construct_pandas_gcs_datasource_without_gcs_options():
     assert pandas_gcs_datasource.name == "pandas_gcs_datasource"
 
 
-@pytest.mark.unit
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.oauth2.service_account.Credentials.from_service_account_file")
 @mock.patch("google.cloud.storage.Client")
@@ -118,9 +118,9 @@ def test_construct_pandas_gcs_datasource_with_filename_in_gcs_options(
     assert pandas_gcs_datasource.name == "pandas_gcs_datasource"
 
 
-@pytest.mark.unit
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.oauth2.service_account.Credentials.from_service_account_info")
 @mock.patch("google.cloud.storage.Client")
@@ -139,9 +139,9 @@ def test_construct_pandas_gcs_datasource_with_info_in_gcs_options(
     assert pandas_gcs_datasource.name == "pandas_gcs_datasource"
 
 
-@pytest.mark.unit
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_add_csv_asset_to_datasource(
@@ -157,9 +157,9 @@ def test_add_csv_asset_to_datasource(
     assert asset.name == "csv_asset"
 
 
-@pytest.mark.unit
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_keys: List[str]):
@@ -170,9 +170,9 @@ def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_ke
     assert asset.name == "csv_asset"
 
 
-@pytest.mark.unit
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_csv_asset_with_batching_regex_named_parameters(
@@ -192,9 +192,9 @@ def test_csv_asset_with_batching_regex_named_parameters(
     assert options == ("path", "year", "month")
 
 
-@pytest.mark.unit
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_csv_asset_with_non_string_batching_regex_named_parameters(
@@ -212,9 +212,9 @@ def test_csv_asset_with_non_string_batching_regex_named_parameters(
         asset.build_batch_request({"name": "alex", "timestamp": "1234567890", "price": 1300})
 
 
-@pytest.mark.unit
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_add_csv_asset_with_recursive_file_discovery_to_datasource(

tests/datasource/fluent/test_spark_google_cloud_storage_datasource.py

@@ -81,7 +81,7 @@ def object_keys() -> List[str]:
 
 @pytest.fixture
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 def csv_asset(
     mock_list_keys,
@@ -95,7 +95,7 @@ def csv_asset(
     return asset
 
 
-@pytest.mark.big
+@pytest.mark.gcs_deps
 def test_construct_spark_gcs_datasource_without_gcs_options():
     google_cred_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
     if not google_cred_file:
@@ -112,9 +112,9 @@ def test_construct_spark_gcs_datasource_without_gcs_options():
     assert spark_gcs_datasource.name == "spark_gcs_datasource"
 
 
-@pytest.mark.big
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.oauth2.service_account.Credentials.from_service_account_file")
 @mock.patch("google.cloud.storage.Client")
@@ -134,9 +134,9 @@ def test_construct_spark_gcs_datasource_with_filename_in_gcs_options(
     assert spark_gcs_datasource.name == "spark_gcs_datasource"
 
 
-@pytest.mark.big
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.oauth2.service_account.Credentials.from_service_account_info")
 @mock.patch("google.cloud.storage.Client")
@@ -156,9 +156,9 @@ def test_construct_spark_gcs_datasource_with_info_in_gcs_options(
     assert spark_gcs_datasource.name == "spark_gcs_datasource"
 
 
-@pytest.mark.big
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_add_csv_asset_to_datasource(
@@ -177,9 +177,9 @@ def test_add_csv_asset_to_datasource(
     assert asset.batch_metadata == asset_specified_metadata
 
 
-@pytest.mark.big
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_keys: List[str]):
@@ -190,9 +190,9 @@ def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_ke
     assert asset.name == "csv_asset"
 
 
-@pytest.mark.big
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_csv_asset_with_batching_regex_named_parameters(
@@ -211,9 +211,9 @@ def test_csv_asset_with_batching_regex_named_parameters(
     assert options == ("path", "year", "month")
 
 
-@pytest.mark.big
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_csv_asset_with_non_string_batching_regex_named_parameters(
@@ -235,9 +235,9 @@ def test_csv_asset_with_non_string_batching_regex_named_parameters(
         )
 
 
-@pytest.mark.big
+@pytest.mark.gcs_deps
 @mock.patch(
-    "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
+    "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys"
 )
 @mock.patch("google.cloud.storage.Client")
 def test_add_csv_asset_with_recursive_file_discovery_to_datasource(