diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6fd31a14321..c220d286adbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -535,6 +535,7 @@ jobs: - bigquery - databricks - filesystem + - gcs_deps - sql_server - mysql - postgresql diff --git a/pyproject.toml b/pyproject.toml index 6f75aacd9501..b4b9ca95dd92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -692,6 +692,7 @@ markers = [ "docs: mark a test as a docs test.", "external_sqldialect: mark test as requiring install of an external sql dialect.", "filesystem: mark tests using the filesystem as the storage backend.", + "gcs_deps: mark tests that need google cloud storage dependencies but no credentials", "integration: mark test as an integration test.", "sql_server: mark a test as SQL Server-dependent.", "mysql: mark a test as mysql-dependent.", diff --git a/reqs/requirements-dev-gcs.txt b/reqs/requirements-dev-gcs.txt new file mode 100644 index 000000000000..8606a1480803 --- /dev/null +++ b/reqs/requirements-dev-gcs.txt @@ -0,0 +1,3 @@ +# needed to prevent excessive pip backtracking on python 3.11 +google-cloud-storage>=2.10.0;python_version >= '3.11' +google-cloud-storage>=1.28.0;python_version < '3.11' diff --git a/tasks.py b/tasks.py index b9ec2b16a956..7dd9eb050fa1 100644 --- a/tasks.py +++ b/tasks.py @@ -879,6 +879,7 @@ class TestDependencies(NamedTuple): services=("spark",), extra_pytest_args=("--spark", "--docs-tests"), ), + "gcs_deps": TestDependencies(("reqs/requirements-dev-gcs.txt",)), "gx-redshift": TestDependencies( requirement_files=("reqs/requirements-dev-gx-redshift.txt",), ), diff --git a/tests/conftest.py b/tests/conftest.py index 2e7b36c8f6aa..0b470e2c7edd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -121,6 +121,7 @@ "databricks", "docs", "filesystem", + "gcs_deps", "generic_sql", "integration", "mysql", diff --git a/tests/datasource/fluent/test_pandas_google_cloud_storage_datasource.py b/tests/datasource/fluent/test_pandas_google_cloud_storage_datasource.py index ae0dd5abe425..7f8d05c4bf8f 100644 --- a/tests/datasource/fluent/test_pandas_google_cloud_storage_datasource.py +++ b/tests/datasource/fluent/test_pandas_google_cloud_storage_datasource.py @@ -81,7 +81,7 @@ def object_keys() -> List[str]: ] -@pytest.mark.unit +@pytest.mark.gcs_deps def test_construct_pandas_gcs_datasource_without_gcs_options(): google_cred_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") if not google_cred_file: @@ -97,9 +97,9 @@ def test_construct_pandas_gcs_datasource_without_gcs_options(): assert pandas_gcs_datasource.name == "pandas_gcs_datasource" -@pytest.mark.unit +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.oauth2.service_account.Credentials.from_service_account_file") @mock.patch("google.cloud.storage.Client") @@ -118,9 +118,9 @@ def test_construct_pandas_gcs_datasource_with_filename_in_gcs_options( assert pandas_gcs_datasource.name == "pandas_gcs_datasource" -@pytest.mark.unit +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.oauth2.service_account.Credentials.from_service_account_info") @mock.patch("google.cloud.storage.Client") @@ -139,9 +139,9 @@ def test_construct_pandas_gcs_datasource_with_info_in_gcs_options( assert pandas_gcs_datasource.name == "pandas_gcs_datasource" -@pytest.mark.unit +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_add_csv_asset_to_datasource( @@ -157,9 +157,9 @@ def test_add_csv_asset_to_datasource( assert asset.name == "csv_asset" -@pytest.mark.unit +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_keys: List[str]): @@ -170,9 +170,9 @@ def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_ke assert asset.name == "csv_asset" -@pytest.mark.unit +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_csv_asset_with_batching_regex_named_parameters( @@ -192,9 +192,9 @@ def test_csv_asset_with_batching_regex_named_parameters( assert options == ("path", "year", "month") -@pytest.mark.unit +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_csv_asset_with_non_string_batching_regex_named_parameters( @@ -212,9 +212,9 @@ def test_csv_asset_with_non_string_batching_regex_named_parameters( asset.build_batch_request({"name": "alex", "timestamp": "1234567890", "price": 1300}) -@pytest.mark.unit +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_add_csv_asset_with_recursive_file_discovery_to_datasource( diff --git a/tests/datasource/fluent/test_spark_google_cloud_storage_datasource.py b/tests/datasource/fluent/test_spark_google_cloud_storage_datasource.py index 5b49a4b49b8c..62162aaeb950 100644 --- a/tests/datasource/fluent/test_spark_google_cloud_storage_datasource.py +++ b/tests/datasource/fluent/test_spark_google_cloud_storage_datasource.py @@ -81,7 +81,7 @@ def object_keys() -> List[str]: @pytest.fixture @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) def csv_asset( mock_list_keys, @@ -95,7 +95,7 @@ def csv_asset( return asset -@pytest.mark.big +@pytest.mark.gcs_deps def test_construct_spark_gcs_datasource_without_gcs_options(): google_cred_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") if not google_cred_file: @@ -112,9 +112,9 @@ def test_construct_spark_gcs_datasource_without_gcs_options(): assert spark_gcs_datasource.name == "spark_gcs_datasource" -@pytest.mark.big +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.oauth2.service_account.Credentials.from_service_account_file") @mock.patch("google.cloud.storage.Client") @@ -134,9 +134,9 @@ def test_construct_spark_gcs_datasource_with_filename_in_gcs_options( assert spark_gcs_datasource.name == "spark_gcs_datasource" -@pytest.mark.big +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.oauth2.service_account.Credentials.from_service_account_info") @mock.patch("google.cloud.storage.Client") @@ -156,9 +156,9 @@ def test_construct_spark_gcs_datasource_with_info_in_gcs_options( assert spark_gcs_datasource.name == "spark_gcs_datasource" -@pytest.mark.big +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_add_csv_asset_to_datasource( @@ -177,9 +177,9 @@ def test_add_csv_asset_to_datasource( assert asset.batch_metadata == asset_specified_metadata -@pytest.mark.big +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_keys: List[str]): @@ -190,9 +190,9 @@ def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_ke assert asset.name == "csv_asset" -@pytest.mark.big +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_csv_asset_with_batching_regex_named_parameters( @@ -211,9 +211,9 @@ def test_csv_asset_with_batching_regex_named_parameters( assert options == ("path", "year", "month") -@pytest.mark.big +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_csv_asset_with_non_string_batching_regex_named_parameters( @@ -235,9 +235,9 @@ def test_csv_asset_with_non_string_batching_regex_named_parameters( ) -@pytest.mark.big +@pytest.mark.gcs_deps @mock.patch( - "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" + "great_expectations.datasource.fluent.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") def test_add_csv_asset_with_recursive_file_discovery_to_datasource(