diff --git a/CHANGELOG.md b/CHANGELOG.md index eaccd38b..27392f61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,13 @@ Types of changes # Latch SDK Changelog +## 2.56.0 - 2025-02-10 + +### Changed + +* Update `LPath` to use `latch_persistence` instead of `latch cp` infrastructure + * allows for dramatic download speed ups due to multiprocessing the downloads into chunks + ## 2.55.3 - 2025-01-23 ### Changed diff --git a/pyproject.toml b/pyproject.toml index 81b288bb..6996960c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ include = ["src/**/*.py", "src/latch_cli/services/init/*"] [project] name = "latch" -version = "2.55.3" +version = "2.56.0" description = "The Latch SDK" authors = [{ name = "Kenny Workman", email = "kenny@latch.bio" }] maintainers = [ @@ -52,6 +52,7 @@ dependencies = [ "setuptools>=75.3.0", "pyxattr>=0.8.1", "orjson>=3.10.12", + "latch-persistence>=0.1.1", ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/src/latch/ldata/path.py b/src/latch/ldata/path.py index 9e072132..fb3d254e 100644 --- a/src/latch/ldata/path.py +++ b/src/latch/ldata/path.py @@ -3,6 +3,7 @@ import re import shutil import sys +import warnings from collections.abc import Iterator from dataclasses import dataclass, field from pathlib import Path @@ -20,16 +21,13 @@ Scalar, ) from flytekit.extend import TypeEngine, TypeTransformer -from typing_extensions import Self +from latch_persistence import LatchPersistence from latch.ldata.type import LatchPathError, LDataNodeType from latch_cli.utils import urljoins -from ._transfer.download import download as _download from ._transfer.node import get_node_data as _get_node_data -from ._transfer.progress import Progress as _Progress from ._transfer.remote_copy import remote_copy as _remote_copy -from ._transfer.upload import upload as _upload from ._transfer.utils import query_with_retry node_id_regex = re.compile(r"^latch://(?P[0-9]+)\.node$") @@ -73,6 +71,14 @@ class LPath: default_factory=_Cache, init=False, repr=False, hash=False, compare=False ) + _persistence: LatchPersistence = field( + default_factory=LatchPersistence, + init=False, + repr=False, + hash=False, + compare=False, + ) + path: str def __post_init__(self): @@ -199,7 +205,7 @@ def version_id(self, *, load_if_missing: bool = True) -> Optional[str]: def is_dir(self, *, load_if_missing: bool = True) -> bool: return self.type(load_if_missing=load_if_missing) in _dir_types - def iterdir(self) -> Iterator[Self]: + def iterdir(self) -> Iterator["LPath"]: """Yield LPaths objects contained within the directory. Should only be called on directories. Does not recursively list directories. @@ -289,13 +295,17 @@ def upload_from(self, src: Path, *, show_progress_bar: bool = False) -> None: src: The source path. show_progress_bar: Whether to show a progress bar during the upload. """ - _upload( - os.fspath(src), - self.path, - progress=_Progress.tasks if show_progress_bar else _Progress.none, - verbose=False, - create_parents=True, - ) + if show_progress_bar: + warnings.warn( + "argument `show_progress_bar` is deprecated and will be removed in a future release of `latch`.", + stacklevel=2, + ) + + if src.is_dir(): + self._persistence.upload_directory(str(src), self.path) + else: + self._persistence.upload(str(src), self.path) + self._clear_cache() def download( @@ -312,6 +322,12 @@ def download( downloaded there. The temprary directory is deleted when the program exits. show_progress_bar: Whether to show a progress bar during the download. """ + if show_progress_bar: + warnings.warn( + "argument `show_progress_bar` is deprecated and will be removed in a future release of `latch`.", + stacklevel=2, + ) + if dst is None: global _download_idx tmp_dir = Path.home() / ".latch" / "lpath" / str(_download_idx) @@ -336,13 +352,10 @@ def download( ): return dst - _download( - self.path, - dst, - progress=_Progress.tasks if show_progress_bar else _Progress.none, - verbose=False, - confirm_overwrite=False, - ) + if self.is_dir(): + self._persistence.download_directory(self.path, str(dst)) + else: + self._persistence.download(self.path, str(dst)) if not_windows and version_id is not None: xattr.setxattr(dst_str, "user.version_id", version_id) diff --git a/uv.lock b/uv.lock index 4ea2284c..03a8cfa5 100644 --- a/uv.lock +++ b/uv.lock @@ -1005,7 +1005,7 @@ wheels = [ [[package]] name = "latch" -version = "2.54.10" +version = "2.55.3" source = { editable = "." } dependencies = [ { name = "aioconsole" }, @@ -1018,6 +1018,7 @@ dependencies = [ { name = "gql" }, { name = "graphql-core" }, { name = "kubernetes" }, + { name = "latch-persistence" }, { name = "lytekit" }, { name = "lytekitplugins-pods" }, { name = "orjson" }, @@ -1070,6 +1071,7 @@ requires-dist = [ { name = "gql", specifier = "==3.4.0" }, { name = "graphql-core", specifier = "==3.2.3" }, { name = "kubernetes", specifier = ">=24.2.0" }, + { name = "latch-persistence", specifier = ">=0.1.1" }, { name = "lytekit", specifier = "==0.15.13" }, { name = "lytekitplugins-pods", specifier = "==0.6.2" }, { name = "orjson", specifier = ">=3.10.12" }, @@ -1104,6 +1106,18 @@ docs = [ { name = "sphinx-copybutton" }, ] +[[package]] +name = "latch-persistence" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/55/452b5d23c0f9c161fff5f36d4a2e735027455a6a1ecb454c5ed84a9da697/latch_persistence-0.1.1.tar.gz", hash = "sha256:692de1b744f99b3ab50db32d49ab8ec9fedbdf4f1008e1f4251c5460e0683168", size = 9447 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/ee/6713d9174e9e7e18c3c750b767cb2564c15f757fe8673b91d43f430a06e6/latch_persistence-0.1.1-py3-none-any.whl", hash = "sha256:0565978613a9d6d006a21b6a13349bbf78f7a3b6b02dc61dab120567192582cc", size = 5446 }, +] + [[package]] name = "lyteidl" version = "0.2.1"