chore(bigframes): optimize system test teardown (#17443)

brings down bigframes system tests from 23 mins to 12 mins i.e. 2x speed
up.

Author: ohmayr

packages/bigframes/bigframes/session/anonymous_dataset.py

@@ -16,6 +16,7 @@
 import threading
 import uuid
 import warnings
+from concurrent.futures import ThreadPoolExecutor
 from typing import List, Optional, Sequence
 
 import google.cloud.bigquery as bigquery
@@ -170,9 +171,19 @@ def _cleanup_old_udfs(self):
 
     def close(self):
         """Delete tables that were created with this session's session_id."""
-        for table_ref in self._table_ids:
-            self.bqclient.delete_table(table_ref, not_found_ok=True)
-        self._table_ids.clear()
+        if self._table_ids:
+            try:
+                with ThreadPoolExecutor() as executor:
+                    futures = [
+                        executor.submit(
+                            self.bqclient.delete_table, table_ref, not_found_ok=True
+                        )
+                        for table_ref in self._table_ids
+                    ]
+                    for future in futures:
+                        future.result()
+            finally:
+                self._table_ids.clear()
 
         try:
             # Before closing the session, attempt to clean up any uncollected,

packages/bigframes/noxfile.py

@@ -354,6 +354,7 @@ def run_system(
         "py.test",
         "-v",
         f"-n={num_workers}",
+        "--dist=worksteal",
         # Any individual test taking longer than 15 mins will be terminated.
         f"--timeout={timeout_seconds}",
         # Log 20 slowest tests