Use new URLPath instead of PathProxy

Author: aron

README.md

@@ -525,9 +525,9 @@ To use a model:
 > For now `use()` MUST be called in the top level module scope. We may relax this in future.
 
 ```py
-from replicate import use
+import replicate
 
-flux_dev = use("black-forest-labs/flux-dev")
+flux_dev = replicate.use("black-forest-labs/flux-dev")
 outputs = flux_dev(prompt="a cat wearing an amusing hat")
 
 for output in outputs:
@@ -538,7 +538,7 @@ Models that output iterators will return iterators:
 
 
 ```py
-claude = use("anthropic/claude-4-sonnet")
+claude = replicate.use("anthropic/claude-4-sonnet")
 
 output = claude(prompt="Give me a recipe for tasty smashed avocado on sourdough toast that could feed all of California.")
 
@@ -555,10 +555,10 @@ str(output) # "Here's a recipe to feed all of California (about 39 million peopl
 You can pass the results of one model directly into another:
 
 ```py
-from replicate import use
+import replicate
 
-flux_dev = use("black-forest-labs/flux-dev")
-claude = use("anthropic/claude-4-sonnet")
+flux_dev = replicate.use("black-forest-labs/flux-dev")
+claude = replicate.use("anthropic/claude-4-sonnet")
 
 images = flux_dev(prompt="a cat wearing an amusing hat")
 
@@ -570,7 +570,7 @@ print(str(result)) # "This shows an image of a cat wearing a hat ..."
 To create an individual prediction that has not yet resolved, use the `create()` method:
 
 ```
-claude = use("anthropic/claude-4-sonnet")
+claude = replicate.use("anthropic/claude-4-sonnet")
 
 prediction = claude.create(prompt="Give me a recipe for tasty smashed avocado on sourdough toast that could feed all of California.")
 
@@ -579,13 +579,49 @@ prediction.logs() # get current logs (WIP)
 prediction.output() # get the output
 ```
 
-You can access the underlying URL for a Path object returned from a model call by using the `get_path_url()` helper.
+### Downloading file outputs
+
+Output files are provided as Python [os.PathLike](https://docs.python.org/3.12/library/os.html#os.PathLike) objects. These are supported by most of the Python standard library like `open()` and `Path`, as well as third-party libraries like `pillow` and `ffmpeg-python`.
+
+The first time the file is accessed it will be downloaded to a temporary directory on disk ready for use.
+
+Here's an example of how to use the `pillow` package to convert file outputs:
 
 ```py
-from replicate import use
-from replicate.use import get_url_path
+import replicate
+from PIL import Image
+
+flux_dev = replicate.use("black-forest-labs/flux-dev")
+
+images = flux_dev(prompt="a cat wearing an amusing hat")
+for i, path in enumerate(images):
+    with Image.open(path) as img:
+        img.save(f"./output_{i}.png", format="PNG")
+```
+
+For libraries that do not support `Path` or `PathLike` instances you can use `open()` as you would with any other file. For example to use `requests` to upload the file to a different location:
+
+```py
+import replicate
+import requests
+
+flux_dev = replicate.use("black-forest-labs/flux-dev")
+
+images = flux_dev(prompt="a cat wearing an amusing hat")
+for path in images:
+    with open(path, "rb") as f:
+        r = requests.post("https://api.example.com/upload", files={"file": f})
+```
+
+### Accessing outputs as HTTPS URLs
+
+If you do not need to download the output to disk. You can access the underlying URL for a Path object returned from a model call by using the `get_path_url()` helper.
+
+```py
+import replicate
+from replicate import get_url_path
 
-flux_dev = use("black-forest-labs/flux-dev")
+flux_dev = replicate.use("black-forest-labs/flux-dev")
 outputs = flux_dev(prompt="a cat wearing an amusing hat")
 
 for output in outputs:

replicate/__init__.py

@@ -1,7 +1,7 @@
 from replicate.client import Client
 from replicate.pagination import async_paginate as _async_paginate
 from replicate.pagination import paginate as _paginate
-from replicate.use import use
+from replicate.use import get_path_url, use
 
 __all__ = [
     "Client",
@@ -21,6 +21,7 @@
     "trainings",
     "webhooks",
     "default_client",
+    "get_path_url",
 ]
 
 default_client = Client()

replicate/use.py

@@ -2,6 +2,7 @@
 # - [ ] Support text streaming
 # - [ ] Support file streaming
 # - [ ] Support asyncio variant
+import hashlib
 import inspect
 import os
 import sys
@@ -138,7 +139,7 @@ def _process_iterator_item(item: Any, openapi_schema: dict) -> Any:
         # If items are file URLs, download them
         if items_schema.get("type") == "string" and items_schema.get("format") == "uri":
             if isinstance(item, str) and item.startswith(("http://", "https://")):
-                return PathProxy(item)
+                return URLPath(item)
 
     return item
 
@@ -154,7 +155,7 @@ def _process_output_with_schema(output: Any, openapi_schema: dict) -> Any:
     # Handle direct string with format=uri
     if output_schema.get("type") == "string" and output_schema.get("format") == "uri":
         if isinstance(output, str) and output.startswith(("http://", "https://")):
-            return PathProxy(output)
+            return URLPath(output)
         return output
 
     # Handle array of strings with format=uri
@@ -163,7 +164,7 @@ def _process_output_with_schema(output: Any, openapi_schema: dict) -> Any:
         if items.get("type") == "string" and items.get("format") == "uri":
             if isinstance(output, list):
                 return [
-                    PathProxy(url)
+                    URLPath(url)
                     if isinstance(url, str) and url.startswith(("http://", "https://"))
                     else url
                     for url in output
@@ -187,15 +188,15 @@ def _process_output_with_schema(output: Any, openapi_schema: dict) -> Any:
                     if isinstance(value, str) and value.startswith(
                         ("http://", "https://")
                     ):
-                        result[prop_name] = PathProxy(value)
+                        result[prop_name] = URLPath(value)
 
                 # Array of files property
                 elif prop_schema.get("type") == "array":
                     items = prop_schema.get("items", {})
                     if items.get("type") == "string" and items.get("format") == "uri":
                         if isinstance(value, list):
                             result[prop_name] = [
-                                PathProxy(url)
+                                URLPath(url)
                                 if isinstance(url, str)
                                 and url.startswith(("http://", "https://"))
                                 else url
@@ -233,49 +234,52 @@ def __str__(self) -> str:
             return str(self.iterator_factory())
 
 
-class PathProxy(Path):
-    def __init__(self, target: str) -> None:
-        path: Path | None = None
-
-        def ensure_path() -> Path:
-            nonlocal path
-            if path is None:
-                path = _download_file(target)
-            return path
+class URLPath(os.PathLike):
+    """
+    A PathLike that defers filesystem ops until first use. Can be used with
+    most Python file interfaces like `open()` and `pathlib.Path()`.
+    See: https://docs.python.org/3.12/library/os.html#os.PathLike
+    """
 
-        object.__setattr__(self, "__replicate_target__", target)
-        object.__setattr__(self, "__replicate_path__", ensure_path)
+    def __init__(self, url: str) -> None:
+        # store the original URL
+        self.__url__ = url
 
-    def __getattribute__(self, name) -> Any:
-        if name in ("__replicate_path__", "__replicate_target__"):
-            return object.__getattribute__(self, name)
+        # compute target path without touching the filesystem
+        base = Path(tempfile.gettempdir())
+        h = hashlib.sha256(self.__url__.encode("utf-8")).hexdigest()[:16]
+        name = Path(httpx.URL(self.__url__).path).name or h
+        self.__path__ = base / h / name
 
-        # TODO: We should cover other common properties on Path...
-        if name == "__class__":
-            return Path
+    def __fspath__(self) -> str:
+        # on first access, create dirs and download if missing
+        if not self.__path__.exists():
+            subdir = self.__path__.parent
+            subdir.mkdir(parents=True, exist_ok=True)
+            if not os.access(subdir, os.W_OK):
+                raise PermissionError(f"Cannot write to {subdir!r}")
 
-        return getattr(object.__getattribute__(self, "__replicate_path__")(), name)
+            with httpx.Client() as client, client.stream("GET", self.__url__) as resp:
+                resp.raise_for_status()
+                with open(self.__path__, "wb") as f:
+                    for chunk in resp.iter_bytes(chunk_size=16_384):
+                        f.write(chunk)
 
-    def __setattr__(self, name, value) -> None:
-        if name in ("__replicate_path__", "__replicate_target__"):
-            raise ValueError()
+        return str(self.__path__)
 
-        object.__setattr__(
-            object.__getattribute__(self, "__replicate_path__")(), name, value
-        )
+    def __str__(self) -> str:
+        return str(self.__path__)
 
-    def __delattr__(self, name) -> None:
-        if name in ("__replicate_path__", "__replicate_target__"):
-            raise ValueError()
-        delattr(object.__getattribute__(self, "__replicate_path__")(), name)
+    def __repr__(self) -> str:
+        return f"<URLPath url={self.__url__!r} path={self.__path__!r}>"
 
 
 def get_path_url(path: Any) -> str | None:
     """
     Return the remote URL (if any) for a Path output from a model.
     """
     try:
-        return object.__getattribute__(path, "__replicate_target__")
+        return object.__getattribute__(path, "__url__")
     except AttributeError:
         return None
 
@@ -385,7 +389,7 @@ def create(self, *_: Input.args, **inputs: Input.kwargs) -> Run[Output]:
         """
         Start a prediction with the specified inputs.
         """
-        # Process inputs to convert concatenate OutputIterators to strings and PathProxy to URLs
+        # Process inputs to convert concatenate OutputIterators to strings and URLPath to URLs
         processed_inputs = {}
         for key, value in inputs.items():
             if isinstance(value, OutputIterator) and value.is_concatenate: