more

Author: sryza

dev/requirements.txt

@@ -27,6 +27,7 @@ pytest-mypy-plugins==1.9.3
 flake8==3.9.0
 # See SPARK-38680.
 pandas-stubs<1.2.0.54
+types-PyYAML
 
 # Documentation (SQL)
 mkdocs

dev/sparktestsupport/modules.py

@@ -555,8 +555,8 @@ def __hash__(self):
         "pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations",
         "pyspark.sql.tests.pandas.test_pandas_udf_window",
         "pyspark.sql.tests.pandas.test_pandas_sqlmetrics",
-        "pyspark.sql.tests.pandas.test_block_connect_access",
-        "pyspark.sql.tests.pipelines.test_blocking_connect_access"
+        "pyspark.sql.tests.pandas.test_converter",
+        "pyspark.sql.tests.pipelines.test_block_connect_access",
         "pyspark.sql.tests.pipelines.test_cli",
         "pyspark.sql.tests.pipelines.test_decorators",
         "pyspark.sql.tests.pipelines.test_graph_element_registry",

python/pyspark/sql/pipelines/block_connect_access.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 from contextlib import contextmanager
-from typing import Generator, Any
+from typing import Callable, Generator, NoReturn
 
 from pyspark.errors import PySparkException
 from pyspark.sql.connect.proto.base_pb2_grpc import SparkConnectServiceStub
@@ -37,11 +37,11 @@ def block_spark_connect_execution_and_analysis() -> Generator[None, None, None]:
     original_getattr = getattr(SparkConnectServiceStub, "__getattribute__")
 
     # Define a new __getattribute__ method that blocks RPC calls
-    def blocked_getattr(self: Any, name: str) -> Any:
+    def blocked_getattr(self: SparkConnectServiceStub, name: str) -> Callable:
         if name not in BLOCKED_RPC_NAMES:
             return original_getattr(self, name)
 
-        def blocked_method(*args, **kwargs):
+        def blocked_method(*args: object, **kwargs: object) -> NoReturn:
             raise PySparkException(
                 errorClass="ATTEMPT_ANALYSIS_IN_PIPELINE_QUERY_FUNCTION",
                 messageParameters={},

python/pyspark/sql/pipelines/cli.py

@@ -22,7 +22,6 @@
     $ bin/spark-pipelines run --spec /path/to/pipeline.yaml
 """
 from contextlib import contextmanager
-from threading import Thread
 import argparse
 import importlib.util
 import os
@@ -253,7 +252,8 @@ def run(spec_path: Path, remote: str) -> None:
     )
     init_parser.add_argument(
         "--name",
-        help="Name of the project. A directory with this name will be created underneath the current directory.",
+        help="Name of the project. A directory with this name will be created underneath the "
+        "current directory.",
         required=True,
     )
 

python/pyspark/sql/pipelines/init_cli.py

@@ -69,4 +69,4 @@ def init(name: str) -> None:
 
     print(f"Pipeline project '{name}' created successfully. To run your pipeline:")
     print(f"cd '{name}'")
-    print(f"spark-pipelines run")
+    print("spark-pipelines run")