Skip to content

Commit 07d58d3

Browse files
committed
fix: provision sandbox global storage through graphql
2 parents 2596688 + b2d7979 commit 07d58d3

11 files changed

Lines changed: 209 additions & 35 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
### Features
66

7-
* Apps expose `NetworkVolume` and `GlobalVolume` resources with explicit path-keyed mounts and execution-bound filesystem access.
7+
* Apps expose `NetworkVolume` and `GlobalVolume` resources with explicit path-keyed mounts, lazy create-if-missing provisioning, and execution-bound filesystem access. Global volumes use GraphQL for lookup and creation.
88
* Sandboxes support shared storage references, structured ports, execution details, and lifetime updates through synchronous and asynchronous handles.
99

1010
## [1.12.0](https://github.com/runpod/runpod-python/compare/v1.11.0...v1.12.0) (2026-08-10)

‎README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,11 @@ Queue `.remote()` calls request a synchronous result and poll the same job if th
7777

7878
`NetworkVolume(name_or_id, size=50, datacenter=None, create=True)` references
7979
datacenter-local storage. Apps resolve names and choose a datacenter compatible
80-
with every resource sharing the volume. `GlobalVolume(id)` references existing
81-
global storage without imposing a network-volume datacenter constraint. Both
82-
inherit from the abstract `Volume` base.
80+
with every resource sharing the volume. `GlobalVolume(name_or_id, create=True)`
81+
references global storage without a datacenter constraint. Both inherit from
82+
the abstract `Volume` base, resolve by name or ID, and create missing storage
83+
when provisioning remote compute. Set `create=False` to require existing storage.
84+
Global-volume lookup and creation use GraphQL; network volumes use REST.
8385

8486
Declare attachments with `mounts={"/path": volume}`. Tasks support one network
8587
and one global volume at distinct, non-overlapping paths. Queue and API resources

‎runpod/api/mutations/apps.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
"""GraphQL mutations for capabilities absent from the resource REST API."""
22

3+
MUTATION_CREATE_GLOBAL_VOLUME = """
4+
mutation createGlobalVolume($input: GlobalStoreBucketCreateInput!) {
5+
globalStoreBucketCreate(input: $input) { id name }
6+
}
7+
"""
8+
39
# endpoint saves retain flash bindings, cached models, and schedules atomically.
410
MUTATION_SAVE_ENDPOINT = """
511
mutation saveEndpoint($input: EndpointInput!) {

‎runpod/api/queries/apps.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
"""GraphQL queries for capabilities absent from the resource REST API."""
22

3+
QUERY_GLOBAL_VOLUMES = """
4+
query myGlobalVolumes {
5+
myself {
6+
globalStoreBuckets { id name }
7+
}
8+
}
9+
"""
10+
311
# rest stock filters require a power-of-two count of at least two.
412
QUERY_CPU_STOCK = """
513
query CpuStock($cpuFlavorInput: CpuFlavorInput, $specificsInput: SpecificsInput) {

‎runpod/api/sandboxes.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
from yarl import URL
1616

1717
from runpod import error
18+
from runpod.api.graphql import _build_payload, _check_response
19+
from runpod.api.mutations.apps import MUTATION_CREATE_GLOBAL_VOLUME
20+
from runpod.api.queries.apps import QUERY_GLOBAL_VOLUMES
1821
from runpod.api.rest import (
1922
_build_headers,
2023
_build_url,
@@ -328,6 +331,26 @@ async def _request(
328331
)
329332
return payload
330333

334+
async def _graphql(
335+
self, query: str, variables: dict[str, Any] | None = None
336+
) -> dict[str, Any]:
337+
payload = await self._request(
338+
"POST", "/graphql", body=_build_payload(query, variables)
339+
)
340+
if payload is None:
341+
raise error.QueryError("GraphQL response is empty", query)
342+
return _check_response(payload, query)["data"]
343+
344+
async def list_global_volumes(self) -> list[dict[str, Any]]:
345+
data = await self._graphql(QUERY_GLOBAL_VOLUMES)
346+
return data["myself"]["globalStoreBuckets"]
347+
348+
async def create_global_volume(self, name: str) -> dict[str, Any]:
349+
data = await self._graphql(
350+
MUTATION_CREATE_GLOBAL_VOLUME, {"input": {"name": name}}
351+
)
352+
return data["globalStoreBucketCreate"]
353+
331354
async def list_network_volumes(self) -> list[dict[str, Any]]:
332355
"""list storage through this handle's credentials and control-plane URL."""
333356
payload = await self._request("GET", "/v2/network-volumes")

‎runpod/apps/api.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
"""control-plane calls for app provisioning.
22
33
rest handles equivalent resource operations; graphql handles flash lifecycle,
4-
auth, secrets, task provisioning, and endpoint capabilities absent from rest.
5-
management verbs for the wider sdk stay in runpod.api.ctl_commands.
4+
auth, secrets, global storage, task provisioning, and endpoint capabilities
5+
absent from rest. management verbs for the wider sdk stay in runpod.api.ctl_commands.
66
"""
77

88
from typing import Any, Dict, List, Optional
@@ -387,6 +387,16 @@ async def cpu_stock_status(
387387
)
388388
return _stock_in_datacenter(data, data_center_id)
389389

390+
async def list_global_volumes(self) -> List[Dict[str, Any]]:
391+
data = await self._execute(app_queries.QUERY_GLOBAL_VOLUMES, retry=True)
392+
return data["myself"]["globalStoreBuckets"]
393+
394+
async def create_global_volume(self, name: str) -> Dict[str, Any]:
395+
data = await self._execute(
396+
app_mutations.MUTATION_CREATE_GLOBAL_VOLUME, {"input": {"name": name}}
397+
)
398+
return data["globalStoreBucketCreate"]
399+
390400
async def list_network_volumes(self) -> List[Dict[str, Any]]:
391401
data = await run_rest_request_async(
392402
"GET", "/v2/network-volumes", api_key=self._api_key

‎runpod/apps/volume.py‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -84,19 +84,20 @@ def __repr__(self) -> str:
8484

8585

8686
class GlobalVolume(Volume):
87-
"""an existing global volume identified by its resource id."""
87+
"""a global volume resolved by name or id without datacenter placement."""
8888

8989
kind = "global"
9090

91-
def __init__(self, id: str):
92-
super().__init__(id)
91+
def __init__(self, name: str, *, create: bool = True):
92+
super().__init__(name)
93+
self.create = create
9394

9495
@property
95-
def id(self) -> str:
96+
def name(self) -> str:
9697
return self.reference
9798

9899
def __repr__(self) -> str:
99-
return f"<GlobalVolume {self.id!r}>"
100+
return f"<GlobalVolume {self.name!r}>"
100101

101102

102103
def _mount_path(path: str) -> str:
@@ -183,7 +184,21 @@ async def resolve(
183184
if cached is not None:
184185
return cached
185186
if isinstance(volume, GlobalVolume):
186-
resolved = {"id": volume.id}
187+
client = await self._client()
188+
record = find_by_id_or_name(
189+
await client.list_global_volumes(),
190+
volume.name,
191+
noun="global volumes",
192+
error=VolumeError,
193+
)
194+
if record is None:
195+
if not volume.create:
196+
raise VolumeError(
197+
f"global volume '{volume.name}' not found and create=False"
198+
)
199+
record = await client.create_global_volume(name=volume.name)
200+
log.info("created global volume %s (%s)", volume.name, record["id"])
201+
resolved = {"id": record["id"]}
187202
self._resolved[key] = resolved
188203
return resolved
189204
if not isinstance(volume, NetworkVolume):

‎tests/test_apps/test_api_client.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,16 +59,17 @@ async def test_graphql_errors_propagate_immediately(self):
5959
await client._execute("query {}")
6060
assert transport.await_count == 1
6161

62+
@pytest.mark.parametrize("operation", ["create_app", "create_global_volume"])
6263
@pytest.mark.parametrize(
6364
"failure",
6465
[aiohttp.ClientError("response lost"), OSError("reset")],
6566
)
66-
async def test_mutation_transport_failure_is_not_retried(self, failure):
67+
async def test_mutation_transport_failure_is_not_retried(self, failure, operation):
6768
client = AppsApiClient(api_key="test-key")
6869
transport = AsyncMock(side_effect=failure)
6970
with patch("runpod.apps.api.run_graphql_query_async", transport):
7071
with pytest.raises(type(failure)):
71-
await client.create_app("demo")
72+
await getattr(client, operation)("demo")
7273
assert transport.await_count == 1
7374

7475

‎tests/test_apps/test_tasks.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,9 @@ async def test_start_retries_datacenters_after_capacity_error(self, mounts):
339339
from runpod.error import QueryError
340340

341341
api = MagicMock()
342+
api.list_global_volumes = AsyncMock(
343+
return_value=[{"id": "global-1", "name": "models"}]
344+
)
342345
capacity = "This machine does not have the resources to deploy your pod"
343346
api.deploy_task_pod = AsyncMock(
344347
side_effect=[
@@ -380,9 +383,7 @@ async def test_shared_volume_placement_accounts_for_siblings(self):
380383
from runpod.apps.tasks import TaskExecution
381384

382385
volume = NetworkVolume("shared")
383-
spec = self._spec(
384-
mounts={"/data": volume}, datacenter=["US-IL-1", "EU-RO-1"]
385-
)
386+
spec = self._spec(mounts={"/data": volume}, datacenter=["US-IL-1", "EU-RO-1"])
386387
sibling = ResourceSpec(
387388
kind=ResourceKind.QUEUE,
388389
name="sibling",

‎tests/test_apps/test_volume.py‎

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@ def _spec(name="r", gpu=None, cpu=None):
2323
return ResourceSpec(kind=ResourceKind.TASK, name=name, gpu=gpu, cpu=cpu)
2424

2525

26-
def _api(volumes=None, created=None):
26+
def _api(volumes=None, created=None, global_volumes=None):
2727
api = AsyncMock()
2828
api.list_network_volumes.return_value = volumes or []
29+
api.list_global_volumes.return_value = global_volumes or []
30+
api.create_global_volume.return_value = {"id": "gv-new", "name": "models"}
2931
api.create_network_volume.return_value = created or {
3032
"id": "nv-new",
3133
"name": "models",
@@ -174,26 +176,57 @@ def test_missing_creates_with_placement(self):
174176
assert resolved["id"] == "nv-new"
175177
api.create_network_volume.assert_awaited_once()
176178

177-
def test_missing_no_create_raises(self):
179+
@pytest.mark.parametrize("volume_type", [NetworkVolume, GlobalVolume])
180+
def test_missing_no_create_raises(self, volume_type):
178181
api = _api()
179182
resolver = VolumeResolver(api)
180183
with pytest.raises(VolumeError, match="create=False"):
181184
asyncio.run(
182-
resolver.resolve(
183-
NetworkVolume("models", create=False), [_spec(gpu=None)]
184-
)
185+
resolver.resolve(volume_type("models", create=False), [_spec(gpu=None)])
185186
)
186187

187-
def test_duplicate_names_raise(self):
188+
@pytest.mark.parametrize("volume_type", [NetworkVolume, GlobalVolume])
189+
def test_duplicate_names_raise(self, volume_type):
188190
api = _api(
189191
volumes=[
190192
{"id": "nv-1", "name": "models", "size": 50, "dataCenter": "EU-RO-1"},
191193
{"id": "nv-2", "name": "models", "size": 50, "dataCenter": "US-KS-2"},
192194
]
193195
)
196+
api.list_global_volumes.return_value = api.list_network_volumes.return_value
194197
resolver = VolumeResolver(api)
195198
with pytest.raises(VolumeError, match="reference by id"):
196-
asyncio.run(resolver.resolve(NetworkVolume("models"), [_spec(gpu=None)]))
199+
asyncio.run(resolver.resolve(volume_type("models"), [_spec(gpu=None)]))
200+
201+
@pytest.mark.parametrize("reference", ["models", "gv-1"])
202+
def test_existing_global_resolves_without_creation_or_placement(self, reference):
203+
api = _api(global_volumes=[{"id": "gv-1", "name": "models"}])
204+
resolved = asyncio.run(
205+
VolumeResolver(api).resolve(GlobalVolume(reference, create=False))
206+
)
207+
assert resolved == {"id": "gv-1"}
208+
api.create_global_volume.assert_not_awaited()
209+
api.list_network_volumes.assert_not_awaited()
210+
211+
def test_global_creation_is_cached_and_mounts_resolve_by_name_and_id(
212+
self, tmp_path
213+
):
214+
(tmp_path / "model").write_text("global weights")
215+
api = _api()
216+
resolver = VolumeResolver(api)
217+
218+
async def run():
219+
bindings = await resolver.resolve_mounts(
220+
{str(tmp_path): GlobalVolume("models")}
221+
)
222+
assert await resolver.resolve(GlobalVolume("models")) == {"id": "gv-new"}
223+
return bindings
224+
225+
_configure_mounts(asyncio.run(run()))
226+
assert (GlobalVolume("models").path / "model").read_text() == "global weights"
227+
assert (GlobalVolume("gv-new").path / "model").read_text() == "global weights"
228+
api.create_global_volume.assert_awaited_once()
229+
api.list_network_volumes.assert_not_awaited()
197230

198231
def test_resolution_cached_per_name(self):
199232
api = _api(
@@ -214,7 +247,7 @@ def test_same_reference_in_different_backends_does_not_share_cache(self):
214247
resolver = VolumeResolver(
215248
_api(volumes=[{"id": "nv-1", "name": "models", "dataCenter": "EU-RO-1"}])
216249
)
217-
assert asyncio.run(resolver.resolve(GlobalVolume("models"))) == {"id": "models"}
250+
assert asyncio.run(resolver.resolve(GlobalVolume("models"))) == {"id": "gv-new"}
218251
assert asyncio.run(resolver.resolve(NetworkVolume("models"))) == {
219252
"id": "nv-1",
220253
"dataCenterId": "EU-RO-1",
@@ -244,9 +277,10 @@ def test_pod_pins_to_volume_dc(self):
244277
from runpod.apps.tasks import TaskExecution
245278

246279
api = _api(
280+
global_volumes=[{"id": "gv-1", "name": "shared-global"}],
247281
volumes=[
248282
{"id": "nv-1", "name": "models", "size": 50, "dataCenter": "EU-RO-1"}
249-
]
283+
],
250284
)
251285
spec = ResourceSpec(
252286
kind=ResourceKind.TASK,
@@ -298,18 +332,18 @@ def test_global_attachment_does_not_pin_datacenter(self):
298332

299333
app = App("global-model")
300334

301-
@app.queue(gpu="4090", mounts={"/runpod-volume": GlobalVolume("gv-1")})
335+
@app.queue(gpu="4090", mounts={"/runpod-volume": GlobalVolume("models")})
302336
def generate():
303337
return None
304338

305339
payload = {"locations": "EU-RO-1,US-KS-2"}
306340
asyncio.run(
307-
attach_endpoint_volumes(payload, generate.spec, VolumeResolver(), app)
341+
attach_endpoint_volumes(payload, generate.spec, VolumeResolver(_api()), app)
308342
)
309343
assert payload["locations"] == "EU-RO-1,US-KS-2"
310344
assert payload["networkVolumeIds"] == []
311345
assert payload["volumes"] == [
312-
{"volumeId": "gv-1", "volumeType": "OBJECT_STORE_VOLUME"}
346+
{"volumeId": "gv-new", "volumeType": "OBJECT_STORE_VOLUME"}
313347
]
314348

315349
def test_removing_storage_clears_backend_and_worker_bindings(self):

0 commit comments

Comments
 (0)