Feat: Add new configurable env var AIRBYTE_TEMP_FILE_CLEANUP (#336)

airbyte/caches/base.py

@@ -21,7 +21,7 @@
 from airbyte._future_cdk.state_writers import StdOutStateWriter
 from airbyte.caches._catalog_backend import CatalogBackendBase, SqlCatalogBackend
 from airbyte.caches._state_backend import SqlStateBackend
-from airbyte.constants import DEFAULT_ARROW_MAX_CHUNK_SIZE
+from airbyte.constants import DEFAULT_ARROW_MAX_CHUNK_SIZE, TEMP_FILE_CLEANUP
 from airbyte.datasets._sql import CachedDataset
 
 
@@ -49,7 +49,7 @@ class CacheBase(SqlConfig):
     cache_dir: Path = Field(default=Path(".cache"))
     """The directory to store the cache in."""
 
-    cleanup: bool = True
+    cleanup: bool = TEMP_FILE_CLEANUP
     """Whether to clean up the cache after use."""
 
     _deployed_api_root: Optional[str] = PrivateAttr(default=None)

airbyte/constants.py

@@ -90,3 +90,18 @@ def _get_logging_root() -> Path | None:
 temporary directory. If the directory cannot be created, PyAirbyte will log a warning and
 set this value to `None`.
 """
+
+TEMP_FILE_CLEANUP = bool(
+    os.getenv(
+        key="AIRBYTE_TEMP_FILE_CLEANUP",
+        default="true",
+    )
+    .lower()
+    .replace("false", "")
+    .replace("0", "")
+)
+"""Whether to clean up temporary files after use.
+
+This value is read from the `AIRBYTE_TEMP_FILE_CLEANUP` environment variable. If the variable is
+not set, the default value is `True`.
+"""