Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/+graphql-pagination-variables.changed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Paginated queries generated by `all()`, `filters()`, `get()` and resource pool allocation lookups now pass `offset` and `limit` as GraphQL variables instead of inlining them in the query text. The query document stays identical across pages, allowing the Infrahub server to reuse its cached query analysis, and the query is now rendered once per call instead of once per page. `generate_query_data` also accepts variable placeholder strings (for example `"$offset"`) for its `offset` and `limit` arguments.
24 changes: 15 additions & 9 deletions docs/docs/python-sdk/sdk_ref/infrahub_sdk/node/node.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ overriding the client default for this request only.
#### `generate_query_data`

```python
generate_query_data(self, filters: dict[str, Any] | None = None, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict]
generate_query_data(self, filters: dict[str, Any] | None = None, offset: int | str | None = None, limit: int | str | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict]
```

Generate the full GraphQL query payload for this node kind.
Expand All @@ -314,8 +314,10 @@ relevant attributes are returned alongside the generic fields.
**Args:**

- `filters`: Filters to apply to the query.
- `offset`: Pagination offset.
- `limit`: Pagination limit.
- `offset`: Pagination offset, either a literal value or a
GraphQL variable placeholder such as ``"$offset"``.
- `limit`: Pagination limit, either a literal value or a
GraphQL variable placeholder such as ``"$limit"``.
- `include`: Attributes or relationships to include.
- `exclude`: Attributes or relationships to exclude.
- `fragment`: When ``True`` and the schema is a generic, emit
Expand Down Expand Up @@ -828,7 +830,7 @@ overriding the client default for this request only.
#### `generate_query_data`

```python
generate_query_data(self, filters: dict[str, Any] | None = None, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict]
generate_query_data(self, filters: dict[str, Any] | None = None, offset: int | str | None = None, limit: int | str | None = None, include: list[str] | None = None, exclude: list[str] | None = None, fragment: bool = False, prefetch_relationships: bool = False, partial_match: bool = False, property: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict]
```

Generate the full GraphQL query payload for this node kind.
Expand All @@ -841,8 +843,10 @@ relevant attributes are returned alongside the generic fields.
**Args:**

- `filters`: Filters to apply to the query.
- `offset`: Pagination offset.
- `limit`: Pagination limit.
- `offset`: Pagination offset, either a literal value or a
GraphQL variable placeholder such as ``"$offset"``.
- `limit`: Pagination limit, either a literal value or a
GraphQL variable placeholder such as ``"$limit"``.
- `include`: Attributes or relationships to include.
- `exclude`: Attributes or relationships to exclude.
- `fragment`: When ``True`` and the schema is a generic, emit
Expand Down Expand Up @@ -1352,7 +1356,7 @@ Return the raw GraphQL payload used to build this node.
#### `generate_query_data_init`

```python
generate_query_data_init(self, filters: dict[str, Any] | None = None, offset: int | None = None, limit: int | None = None, include: list[str] | None = None, exclude: list[str] | None = None, partial_match: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict]
generate_query_data_init(self, filters: dict[str, Any] | None = None, offset: int | str | None = None, limit: int | str | None = None, include: list[str] | None = None, exclude: list[str] | None = None, partial_match: bool = False, order: Order | None = None, include_metadata: bool = False) -> dict[str, Any | dict]
```

Build the top-level ``count``/``edges`` skeleton of a GraphQL query for this kind.
Expand All @@ -1364,8 +1368,10 @@ The returned dict is the outer structure consumed by
**Args:**

- `filters`: Filters to apply to the query.
- `offset`: Pagination offset.
- `limit`: Pagination limit.
- `offset`: Pagination offset, either a literal value or a
GraphQL variable placeholder such as ``"$offset"``.
- `limit`: Pagination limit, either a literal value or a
GraphQL variable placeholder such as ``"$limit"``.
- `include`: Attributes or relationships to include.
- `exclude`: Attributes or relationships to exclude.
- `partial_match`: When ``True``, allow partial matches on filter
Expand Down
76 changes: 46 additions & 30 deletions infrahub_sdk/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1169,24 +1169,32 @@ async def filters( # noqa: C901
filters = kwargs
pagination_size = self.pagination_size

# Pagination is passed as GraphQL variables so the rendered query text stays
# identical across pages and can hit the server-side query cache.
query_data = await InfrahubNode(client=self, schema=schema, branch=branch).generate_query_data(
offset="$offset",
limit="$limit",
filters=filters,
include=include,
exclude=exclude,
fragment=fragment,
prefetch_relationships=prefetch_relationships,
partial_match=partial_match,
property=property,
order=order,
include_metadata=include_metadata,
)
query = Query(query=query_data, name=query_name, variables={"offset": int, "limit": int})
query_str = query.render()

async def process_page(page_offset: int, page_number: int) -> tuple[dict, ProcessRelationsNode]:
"""Process a single page of results."""
query_data = await InfrahubNode(client=self, schema=schema, branch=branch).generate_query_data(
offset=page_offset if offset is None else offset,
limit=limit or pagination_size,
filters=filters,
include=include,
exclude=exclude,
fragment=fragment,
prefetch_relationships=prefetch_relationships,
partial_match=partial_match,
property=property,
order=order,
include_metadata=include_metadata,
)
query = Query(query=query_data, name=query_name)
response = await self.execute_graphql(
query=query.render(),
query=query_str,
variables={
"offset": page_offset if offset is None else offset,
"limit": limit or pagination_size,
},
branch_name=branch,
at=at,
tracker=f"query-{str(schema.kind).lower()}-page{page_number}",
Expand Down Expand Up @@ -2968,24 +2976,32 @@ def filters( # noqa: C901
filters = kwargs
pagination_size = self.pagination_size

# Pagination is passed as GraphQL variables so the rendered query text stays
# identical across pages and can hit the server-side query cache.
query_data = InfrahubNodeSync(client=self, schema=schema, branch=branch).generate_query_data(
offset="$offset",
limit="$limit",
filters=filters,
include=include,
exclude=exclude,
fragment=fragment,
prefetch_relationships=prefetch_relationships,
partial_match=partial_match,
property=property,
order=order,
include_metadata=include_metadata,
)
query = Query(query=query_data, name=query_name, variables={"offset": int, "limit": int})
query_str = query.render()

def process_page(page_offset: int, page_number: int) -> tuple[dict, ProcessRelationsNodeSync]:
"""Process a single page of results."""
query_data = InfrahubNodeSync(client=self, schema=schema, branch=branch).generate_query_data(
offset=page_offset if offset is None else offset,
limit=limit or pagination_size,
filters=filters,
include=include,
exclude=exclude,
fragment=fragment,
prefetch_relationships=prefetch_relationships,
partial_match=partial_match,
property=property,
order=order,
include_metadata=include_metadata,
)
query = Query(query=query_data, name=query_name)
response = self.execute_graphql(
query=query.render(),
query=query_str,
variables={
"offset": page_offset if offset is None else offset,
"limit": limit or pagination_size,
},
branch_name=branch,
at=at,
timeout=timeout,
Expand Down
116 changes: 68 additions & 48 deletions infrahub_sdk/node/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -651,8 +651,8 @@ def _validate_file_object_support(self, message: str) -> None:
def generate_query_data_init(
self,
filters: dict[str, Any] | None = None,
offset: int | None = None,
limit: int | None = None,
offset: int | str | None = None,
limit: int | str | None = None,
include: list[str] | None = None,
exclude: list[str] | None = None,
partial_match: bool = False,
Expand All @@ -667,8 +667,10 @@ def generate_query_data_init(

Args:
filters (dict[str, Any], optional): Filters to apply to the query.
offset (int, optional): Pagination offset.
limit (int, optional): Pagination limit.
offset (int | str, optional): Pagination offset, either a literal value or a
GraphQL variable placeholder such as ``"$offset"``.
limit (int | str, optional): Pagination limit, either a literal value or a
GraphQL variable placeholder such as ``"$limit"``.
include (list[str], optional): Attributes or relationships to include.
exclude (list[str], optional): Attributes or relationships to exclude.
partial_match (bool, optional): When ``True``, allow partial matches on filter
Expand Down Expand Up @@ -1353,8 +1355,8 @@ async def _process_hierarchical_fields(
async def generate_query_data(
self,
filters: dict[str, Any] | None = None,
offset: int | None = None,
limit: int | None = None,
offset: int | str | None = None,
limit: int | str | None = None,
include: list[str] | None = None,
exclude: list[str] | None = None,
fragment: bool = False,
Expand All @@ -1373,8 +1375,10 @@ async def generate_query_data(

Args:
filters (dict[str, Any], optional): Filters to apply to the query.
offset (int, optional): Pagination offset.
limit (int, optional): Pagination limit.
offset (int | str, optional): Pagination offset, either a literal value or a
GraphQL variable placeholder such as ``"$offset"``.
limit (int | str, optional): Pagination limit, either a literal value or a
GraphQL variable placeholder such as ``"$limit"``.
include (list[str], optional): Attributes or relationships to include.
exclude (list[str], optional): Attributes or relationships to exclude.
fragment (bool, optional): When ``True`` and the schema is a generic, emit
Expand Down Expand Up @@ -1850,30 +1854,37 @@ async def get_pool_allocated_resources(self, resource: InfrahubNode) -> list[Inf
graphql_query_name = "InfrahubResourcePoolAllocated"
node_ids_per_kind: dict[str, list[str]] = {}

query = Query(
query={
graphql_query_name: {
"@filters": {
"pool_id": "$pool_id",
"resource_id": "$resource_id",
"offset": "$offset",
"limit": "$limit",
},
"count": None,
"edges": {"node": {"id": None, "kind": None, "branch": None, "identifier": None}},
}
},
name="GetAllocatedResourceForPool",
variables={"pool_id": str, "resource_id": str, "offset": int, "limit": int},
)
query_str = query.render()

has_remaining_items = True
page_number = 1
while has_remaining_items:
page_offset = (page_number - 1) * self._client.pagination_size

query = Query(
query={
graphql_query_name: {
"@filters": {
"pool_id": "$pool_id",
"resource_id": "$resource_id",
"offset": page_offset,
"limit": self._client.pagination_size,
},
"count": None,
"edges": {"node": {"id": None, "kind": None, "branch": None, "identifier": None}},
}
},
name="GetAllocatedResourceForPool",
variables={"pool_id": str, "resource_id": str},
)
response = await self._client.execute_graphql(
query=query.render(),
variables={"pool_id": self.id, "resource_id": resource.id},
query=query_str,
variables={
"pool_id": self.id,
"resource_id": resource.id,
"offset": page_offset,
"limit": self._client.pagination_size,
},
branch_name=self._branch,
tracker=f"get-allocated-resources-page{page_number}",
)
Expand Down Expand Up @@ -2572,8 +2583,8 @@ def _process_hierarchical_fields(
def generate_query_data(
self,
filters: dict[str, Any] | None = None,
offset: int | None = None,
limit: int | None = None,
offset: int | str | None = None,
limit: int | str | None = None,
include: list[str] | None = None,
exclude: list[str] | None = None,
fragment: bool = False,
Expand All @@ -2592,8 +2603,10 @@ def generate_query_data(

Args:
filters (dict[str, Any], optional): Filters to apply to the query.
offset (int, optional): Pagination offset.
limit (int, optional): Pagination limit.
offset (int | str, optional): Pagination offset, either a literal value or a
GraphQL variable placeholder such as ``"$offset"``.
limit (int | str, optional): Pagination limit, either a literal value or a
GraphQL variable placeholder such as ``"$limit"``.
include (list[str], optional): Attributes or relationships to include.
exclude (list[str], optional): Attributes or relationships to exclude.
fragment (bool, optional): When ``True`` and the schema is a generic, emit
Expand Down Expand Up @@ -3072,30 +3085,37 @@ def get_pool_allocated_resources(self, resource: InfrahubNodeSync) -> list[Infra
graphql_query_name = "InfrahubResourcePoolAllocated"
node_ids_per_kind: dict[str, list[str]] = {}

query = Query(
query={
graphql_query_name: {
"@filters": {
"pool_id": "$pool_id",
"resource_id": "$resource_id",
"offset": "$offset",
"limit": "$limit",
},
"count": None,
"edges": {"node": {"id": None, "kind": None, "branch": None, "identifier": None}},
}
},
name="GetAllocatedResourceForPool",
variables={"pool_id": str, "resource_id": str, "offset": int, "limit": int},
)
query_str = query.render()

has_remaining_items = True
page_number = 1
while has_remaining_items:
page_offset = (page_number - 1) * self._client.pagination_size

query = Query(
query={
graphql_query_name: {
"@filters": {
"pool_id": "$pool_id",
"resource_id": "$resource_id",
"offset": page_offset,
"limit": self._client.pagination_size,
},
"count": None,
"edges": {"node": {"id": None, "kind": None, "branch": None, "identifier": None}},
}
},
name="GetAllocatedResourceForPool",
variables={"pool_id": str, "resource_id": str},
)
response = self._client.execute_graphql(
query=query.render(),
variables={"pool_id": self.id, "resource_id": resource.id},
query=query_str,
variables={
"pool_id": self.id,
"resource_id": resource.id,
"offset": page_offset,
"limit": self._client.pagination_size,
},
branch_name=self._branch,
tracker=f"get-allocated-resources-page{page_number}",
)
Expand Down
Loading
Loading