diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 8c8b95875..b0c38e4c3 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -21,7 +21,7 @@ jobs: - name: Install Poetry uses: snok/install-poetry@v1.4 with: - version: 2.1.3 + version: 2.3.2 virtualenvs-create: false - name: Install dependencies run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6b2580860..f67408edf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,7 +22,7 @@ jobs: - name: Install Poetry uses: snok/install-poetry@v1.4 with: - version: 2.1.3 + version: 2.3.2 virtualenvs-create: false - name: Poetry details diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 7b51bcfd6..f5a47b9ad 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -25,7 +25,7 @@ jobs: - name: Install Poetry uses: snok/install-poetry@v1.4 with: - version: 2.1.3 + version: 2.3.2 virtualenvs-create: true virtualenvs-in-project: true diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index 20fdbd812..d03a519b8 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -50,7 +50,7 @@ jobs: - name: Install Poetry uses: snok/install-poetry@v1.4 with: - version: 2.1.3 + version: 2.3.2 virtualenvs-create: false - name: Poetry details diff --git a/.github/workflows/test_docs.yml b/.github/workflows/test_docs.yml index 48edb1ad7..86dacce12 100644 --- a/.github/workflows/test_docs.yml +++ b/.github/workflows/test_docs.yml @@ -20,7 +20,7 @@ jobs: - name: Install Poetry uses: snok/install-poetry@v1.4 with: - version: 2.1.3 + version: 2.3.2 virtualenvs-create: false - name: Install dependencies run: | diff --git a/.github/workflows/type-check.yml b/.github/workflows/type-check.yml index ba8f955a4..fde2476a8 100644 --- a/.github/workflows/type-check.yml +++ b/.github/workflows/type-check.yml @@ -22,7 +22,7 @@ jobs: - name: Install Poetry uses: snok/install-poetry@v1.4 with: - version: 2.1.3 + version: 2.3.2 virtualenvs-create: false - name: Poetry details diff --git a/benchmarks/test_benchmark_alias_lookup.py b/benchmarks/test_benchmark_alias_lookup.py new file mode 100644 index 000000000..7be763c2d --- /dev/null +++ b/benchmarks/test_benchmark_alias_lookup.py @@ -0,0 +1,68 @@ +""" +Micro-benchmark for get_column_name_from_alias and related alias functions. +""" + +import pytest + +from benchmarks.conftest import Author, Book + +pytestmark = pytest.mark.asyncio + + +@pytest.mark.parametrize("num_lookups", [1000, 10000]) +def test_get_column_name_from_alias(benchmark, num_lookups: int) -> None: + """Benchmark get_column_name_from_alias - O(n) linear scan per call.""" + # Get all column aliases for the model + aliases = [col.name for col in Author.ormar_config.table.columns] + + def run() -> None: + for _ in range(num_lookups): + for alias in aliases: + Author.get_column_name_from_alias(alias) + + benchmark(run) + + +@pytest.mark.parametrize("num_lookups", [1000, 10000]) +def test_get_column_name_from_alias_book(benchmark, num_lookups: int) -> None: + """Benchmark on Book model (more fields including FKs).""" + aliases = [col.name for col in Book.ormar_config.table.columns] + + def run() -> None: + for _ in range(num_lookups): + for alias in aliases: + Book.get_column_name_from_alias(alias) + + benchmark(run) + + +@pytest.mark.parametrize("num_lookups", [1000, 10000]) +def test_translate_columns_to_aliases(benchmark, num_lookups: int) -> None: + """Benchmark translate_columns_to_aliases - dict key remapping.""" + + def run() -> None: + for _ in range(num_lookups): + kwargs = {"name": "test", "score": 50, "id": 1} + Author.translate_columns_to_aliases(kwargs) + + benchmark(run) + + +@pytest.mark.parametrize("num_lookups", [1000, 10000]) +def test_translate_aliases_to_columns(benchmark, num_lookups: int) -> None: + """Benchmark translate_aliases_to_columns - reverse remapping.""" + # Get aliases + aliases = { + field.get_alias(): "value" + for field_name, field in Author.ormar_config.model_fields.items() + if field.get_alias() + } + if not aliases: + aliases = {"name": "test", "score": 50, "id": 1} + + def run() -> None: + for _ in range(num_lookups): + kwargs = dict(aliases) + Author.translate_aliases_to_columns(kwargs) + + benchmark(run) diff --git a/benchmarks/test_benchmark_merge.py b/benchmarks/test_benchmark_merge.py new file mode 100644 index 000000000..9b4b3970f --- /dev/null +++ b/benchmarks/test_benchmark_merge.py @@ -0,0 +1,118 @@ +"""Benchmark for nested-join row merging in ``_process_query_result_rows``. + +The workload below targets the path optimized by the in-place index +assignment in ``_merge_items_lists``: a parent that fans out across many +joined rows so the matched branch fires repeatedly during +``_recursive_add``, with both halves of late-round merges accumulating +overlapping child PKs. + +Shape: + + Project (1) -> Task (N, FK) -> Tag (M, m2m, shared across tasks) + +A single ``Project`` produces ``N * M`` result rows on +``select_related(["tasks", "tasks__tags"]).all()``. The pairwise +``_recursive_add`` consolidates the duplicates, and in the deeper rounds +both sides hold overlapping ``Task`` PKs (because every adjacent row +carries the same task with a different tag); inside each task the tag +lists also accumulate and overlap across recursion halves. +""" + +import pytest + +import ormar +from benchmarks.conftest import base_ormar_config + +pytestmark = pytest.mark.asyncio + + +class BenchTag(ormar.Model): + ormar_config = base_ormar_config.copy(tablename="bench_merge_tags") + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=50) + + +class BenchProject(ormar.Model): + ormar_config = base_ormar_config.copy(tablename="bench_merge_projects") + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + + +class BenchTask(ormar.Model): + ormar_config = base_ormar_config.copy(tablename="bench_merge_tasks") + + id: int = ormar.Integer(primary_key=True) + project: BenchProject = ormar.ForeignKey( + BenchProject, index=True, related_name="tasks" + ) + title: str = ormar.String(max_length=100) + tags: list[BenchTag] = ormar.ManyToMany(BenchTag) + + +@pytest.mark.parametrize( + ("num_tasks", "tags_per_task"), + [(5, 5), (10, 10), (20, 10)], +) +async def test_select_related_nested_merge( + aio_benchmark, num_tasks: int, tags_per_task: int +): + """Project -> Tasks (FK) -> Tags (m2m) workload. + + Result row count is ``num_tasks * tags_per_task`` for one project; every + row materializes a duplicate Project with one Task (with one Tag). + The merge path consolidates duplicates pairwise via ``_recursive_add`` + — covers ``_merge_items_lists`` end-to-end through the full join / + materialize / merge pipeline. + """ + project = await BenchProject(name="P").save() + tags = [await BenchTag(name=f"t{i}").save() for i in range(tags_per_task)] + for i in range(num_tasks): + task = await BenchTask(project=project, title=f"T{i}").save() + for tag in tags: + await task.tags.add(tag) + + @aio_benchmark + async def query(): + return await BenchProject.objects.select_related(["tasks", "tasks__tags"]).all() + + result = query() + assert len(result) == 1 + assert len(result[0].tasks) == num_tasks + for task in result[0].tasks: + assert len(task.tags) == tags_per_task + + +@pytest.mark.parametrize("list_size", [10, 50, 100]) +def test_merge_items_lists_pk_overlap(benchmark, list_size: int): + """Microbenchmark for ``_merge_items_lists`` with full PK overlap. + + Constructs two equally sized lists of saved tasks where every entry + in ``current_field`` matches an entry in ``other_value`` by PK. This + is the worst case the per-pair list rebuild used to be O(N) on — K + matches, each filtering an N-element ``value_to_set``. With + ``other_idx`` driving in-place writes the cost drops from O(K·N) to + O(K). + + The benchmark calls the merge classmethod directly so the SA query / + row materialization overhead is excluded — the signal we want is the + inner loop only. Every task is fully populated (no relations to + recurse into) so ``merge_two_instances`` is light and the + ``_merge_items_lists`` body itself dominates. + """ + project = BenchProject(id=1, name="p") + current_field = [ + BenchTask(id=i, project=project, title=f"T{i}") for i in range(list_size) + ] + other_value = [ + BenchTask(id=i, project=project, title=f"T{i}") for i in range(list_size) + ] + + benchmark( + BenchTask._merge_items_lists, + field_name="tasks", + current_field=current_field, + other_value=other_value, + relation_map={"tasks": ...}, + ) diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 1a06f8d36..d008e15ea 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -384,7 +384,7 @@ def expand_relationship( :return: returns untouched value for normal fields, expands only for relations :rtype: Any """ - return value + return value # pragma: no cover def set_self_reference_flag(self) -> None: """ diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index ad00e3f2d..e2f9a29cb 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -2,6 +2,7 @@ import sys import uuid from dataclasses import dataclass +from functools import cached_property from random import choices from typing import TYPE_CHECKING, Any, ForwardRef, Optional, Union, cast, overload @@ -491,28 +492,6 @@ def _extract_model_from_sequence( for val in value ] - def _register_existing_model( - self, value: "Model", child: "Model", to_register: bool - ) -> "Model": - """ - Takes already created instance and registers it for parent. - Registration is mutual, so children have also reference to parent. - - Used in reverse FK relations and normal FK for single models. - - :param value: already instantiated Model - :type value: Model - :param child: child/ related Model - :type child: Model - :param to_register: flag if the relation should be set in RelationshipManager - :type to_register: bool - :return: (if needed) registered Model - :rtype: Model - """ - if to_register: - self.register_relation(model=value, child=child) - return value - def _construct_model_from_dict( self, value: dict, child: "Model", to_register: bool ) -> "Model": @@ -611,6 +590,23 @@ def has_unresolved_forward_refs(self) -> bool: """ return self.to.__class__ == ForwardRef + @cached_property + def _constructor_dispatch(self) -> dict[str, Any]: + """ + Per-field map from input class name to the constructor handling that + shape. Built once at first access — by the time + ``expand_relationship`` runs, ``_verify_model_can_be_initialized`` + has already gated on ``requires_ref_update``, so the bound methods + captured here are stable. + + :return: dispatch table for the slow path of ``expand_relationship`` + :rtype: dict[str, Any] + """ + return { + "dict": self._construct_model_from_dict, + "list": self._extract_model_from_sequence, + } + def expand_relationship( self, value: Any, @@ -636,16 +632,17 @@ def expand_relationship( """ if value is None: return None if not self.virtual else [] - constructors = { - f"{self.to.__name__}": self._register_existing_model, - "dict": self._construct_model_from_dict, - "list": self._extract_model_from_sequence, - } - - model = constructors.get( # type: ignore + # Fast path: ``value`` is already a Model of ``self.to``. Dominant + # case in row materialization and in user kwargs that pass + # constructed Models directly. Skips the dispatch table and the + # ``_register_existing_model`` indirection entirely. + if value.__class__ is self.to: + if to_register: + self.register_relation(model=value, child=cast("Model", child)) + return value + return self._constructor_dispatch.get( value.__class__.__name__, self._construct_model_from_pk )(value, child, to_register) - return model def get_relation_name(self) -> str: # pragma: no cover """ diff --git a/ormar/fields/parsers.py b/ormar/fields/parsers.py index f704e8276..55d83a3de 100644 --- a/ormar/fields/parsers.py +++ b/ormar/fields/parsers.py @@ -1,9 +1,9 @@ -import base64 import datetime import decimal import uuid -from typing import Any, Callable, Optional, Union +from typing import Callable, Optional +import ormar_rust_utils import pydantic from pydantic_core import SchemaValidator, core_schema @@ -27,43 +27,9 @@ def encode_decimal(value: decimal.Decimal, precision: Optional[int] = None) -> f ) -def encode_bytes(value: Union[str, bytes], represent_as_string: bool = False) -> str: - if represent_as_string: - value = ( - value if isinstance(value, str) else base64.b64encode(value).decode("utf-8") - ) - else: - value = value if isinstance(value, str) else value.decode("utf-8") - return value - - -def decode_bytes(value: str, represent_as_string: bool = False) -> bytes: - if represent_as_string: - return value if isinstance(value, bytes) else base64.b64decode(value) - return value if isinstance(value, bytes) else value.encode("utf-8") - - -def encode_json(value: Any) -> Optional[str]: - if isinstance(value, (datetime.date, datetime.datetime, datetime.time)): - value = value.isoformat() - value = json.dumps(value) if not isinstance(value, str) else re_dump_value(value) - value = value.decode("utf-8") if isinstance(value, bytes) else value - return value - - -def re_dump_value(value: str) -> Union[str, bytes]: - """ - Re-dumps value due to different string representation in orjson and json - :param value: string to re-dump - :type value: str - :return: re-dumped value - :rtype: list[str] - """ - try: - result: Union[str, bytes] = json.dumps(json.loads(value)) - except json.JSONDecodeError: - result = value - return result +encode_bytes = ormar_rust_utils.encode_bytes +decode_bytes = ormar_rust_utils.decode_bytes +encode_json = ormar_rust_utils.encode_json ENCODERS_MAP: dict[type, Callable] = { diff --git a/ormar/models/helpers/models.py b/ormar/models/helpers/models.py index b4e333015..13c69b224 100644 --- a/ormar/models/helpers/models.py +++ b/ormar/models/helpers/models.py @@ -1,6 +1,6 @@ -import itertools -from typing import TYPE_CHECKING, Any, ForwardRef +from typing import TYPE_CHECKING, ForwardRef +import ormar_rust_utils import pydantic import ormar # noqa: I100 @@ -123,19 +123,7 @@ def group_related_list(list_: list) -> dict: :return: list converted to dictionary to avoid repetition and group nested models :rtype: dict[str, list] """ - result_dict: dict[str, Any] = dict() - list_.sort(key=lambda x: x.split("__")[0]) - grouped = itertools.groupby(list_, key=lambda x: x.split("__")[0]) - for key, group in grouped: - group_list = list(group) - new = sorted( - ["__".join(x.split("__")[1:]) for x in group_list if len(x.split("__")) > 1] - ) - if any("__" in x for x in new): - result_dict[key] = group_related_list(new) - else: - result_dict.setdefault(key, []).extend(new) - return dict(sorted(result_dict.items(), key=lambda item: len(item[1]))) + return ormar_rust_utils.group_related_list(list_) def config_field_not_set(model: type["Model"], field_name: str) -> bool: diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index dc0dea068..2c8083ad1 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -76,6 +76,12 @@ def add_cached_properties(new_model: type["Model"]) -> None: new_model._json_fields = set() new_model._bytes_fields = set() new_model._onupdate_fields = set() + # Lazy-populated in NewBaseModel._process_kwargs on first init per class. + # ormar_config.extra and ormar_config.model_fields are not finalized at + # this point in class creation, so eager init would be premature. + new_model._pydantic_field_names = None + new_model._extra_is_ignore = None + new_model._allowed_kwarg_names = None def add_property_fields(new_model: type["Model"], attrs: dict) -> None: # noqa: CCR001 diff --git a/ormar/models/mixins/alias_mixin.py b/ormar/models/mixins/alias_mixin.py index 3ed187877..65916db96 100644 --- a/ormar/models/mixins/alias_mixin.py +++ b/ormar/models/mixins/alias_mixin.py @@ -1,5 +1,7 @@ from typing import TYPE_CHECKING +import ormar_rust_utils + class AliasMixin: """ @@ -11,6 +13,27 @@ class AliasMixin: ormar_config: OrmarConfig + _alias_to_field_map: dict[str, str] + _field_to_alias_map: dict[str, str] + + @classmethod + def _build_alias_cache(cls) -> None: + """ + Build and cache alias mappings for this model class. + Builds two dicts: + - _field_to_alias_map: field_name -> db_alias + - _alias_to_field_map: db_alias -> field_name (reverse) + """ + field_to_alias = {} + for field_name, field in cls.ormar_config.model_fields.items(): + alias = field.get_alias() + if alias: + field_to_alias[field_name] = alias + cls._field_to_alias_map = field_to_alias + cls._alias_to_field_map = ormar_rust_utils.build_reverse_alias_map( + field_to_alias + ) + @classmethod def get_column_alias(cls, field_name: str) -> str: """ @@ -21,8 +44,11 @@ def get_column_alias(cls, field_name: str) -> str: :return: alias (db name) if set, otherwise passed name :rtype: str """ - field = cls.ormar_config.model_fields.get(field_name) - return field.get_alias() if field is not None else field_name + try: + return cls._field_to_alias_map.get(field_name, field_name) + except AttributeError: + cls._build_alias_cache() + return cls._field_to_alias_map.get(field_name, field_name) @classmethod def get_column_name_from_alias(cls, alias: str) -> str: @@ -34,10 +60,11 @@ def get_column_name_from_alias(cls, alias: str) -> str: :return: field name if set, otherwise passed alias (db name) :rtype: str """ - for field_name, field in cls.ormar_config.model_fields.items(): - if field.get_alias() == alias: - return field_name - return alias # if not found it's not an alias but actual name + try: + return cls._alias_to_field_map.get(alias, alias) + except AttributeError: + cls._build_alias_cache() + return cls._alias_to_field_map.get(alias, alias) @classmethod def translate_columns_to_aliases(cls, new_kwargs: dict) -> dict: @@ -50,9 +77,15 @@ def translate_columns_to_aliases(cls, new_kwargs: dict) -> dict: :return: dict with aliases and their values :rtype: dict """ - for field_name, field in cls.ormar_config.model_fields.items(): - if field_name in new_kwargs: - new_kwargs[field.get_alias()] = new_kwargs.pop(field_name) + try: + field_to_alias = cls._field_to_alias_map + except AttributeError: + cls._build_alias_cache() + field_to_alias = cls._field_to_alias_map + for field_name in list(new_kwargs.keys()): + alias = field_to_alias.get(field_name) + if alias and alias != field_name: + new_kwargs[alias] = new_kwargs.pop(field_name) return new_kwargs @classmethod @@ -66,7 +99,13 @@ def translate_aliases_to_columns(cls, new_kwargs: dict) -> dict: :return: dict with fields names and their values :rtype: dict """ - for field_name, field in cls.ormar_config.model_fields.items(): - if field.get_alias() and field.get_alias() in new_kwargs: - new_kwargs[field_name] = new_kwargs.pop(field.get_alias()) + try: + alias_to_field = cls._alias_to_field_map + except AttributeError: # pragma: nocover + cls._build_alias_cache() + alias_to_field = cls._alias_to_field_map + for key in list(new_kwargs.keys()): + field_name = alias_to_field.get(key) + if field_name and field_name != key: + new_kwargs[field_name] = new_kwargs.pop(key) return new_kwargs diff --git a/ormar/models/mixins/excludable_mixin.py b/ormar/models/mixins/excludable_mixin.py index eeaa00840..28c65ab7b 100644 --- a/ormar/models/mixins/excludable_mixin.py +++ b/ormar/models/mixins/excludable_mixin.py @@ -55,6 +55,29 @@ def _populate_pk_column( columns.append(pk_alias) return columns + @staticmethod + def _get_table_column_pairs( + model: Union[type["Model"], type["ModelRow"]], + ) -> list[tuple[str, str]]: + """ + Returns cached list of (col_name, field_name) tuples for the model's table. + Built once per model class and cached to avoid repeated SA column iteration. + + :param model: model to get column pairs for + :type model: type["Model"] + :return: list of (column_name, field_name) tuples + :rtype: list[tuple[str, str]] + """ + cached = getattr(model, "_table_column_pairs", None) + if cached is not None: + return cached + pairs = [ + (col.name, model.get_column_name_from_alias(col.name)) + for col in model.ormar_config.table.columns + ] + model._table_column_pairs = pairs # type: ignore[union-attr] + return pairs + @classmethod def own_table_columns( cls, @@ -87,26 +110,17 @@ def own_table_columns( :rtype: list[str] """ model_excludable = excludable.get(model_cls=model, alias=alias) # type: ignore - columns = [ - model.get_column_name_from_alias(col.name) if not use_alias else col.name - for col in model.ormar_config.table.columns - ] - field_names = [ - model.get_column_name_from_alias(col.name) - for col in model.ormar_config.table.columns - ] - if model_excludable.include: - columns = [ - col - for col, name in zip(columns, field_names) - if model_excludable.is_included(name) - ] - if model_excludable.exclude: - columns = [ - col - for col, name in zip(columns, field_names) - if not model_excludable.is_excluded(name) - ] + has_include = bool(model_excludable.include) + has_exclude = bool(model_excludable.exclude) + + column_pairs = cls._get_table_column_pairs(model) + columns = [] + for col_name, field_name in column_pairs: + if has_include and not model_excludable.is_included(field_name): + continue + if has_exclude and model_excludable.is_excluded(field_name): + continue + columns.append(col_name if use_alias else field_name) # always has to return pk column for ormar to work if add_pk_columns: diff --git a/ormar/models/mixins/merge_mixin.py b/ormar/models/mixins/merge_mixin.py index 1117b84b5..5a4bf3fc3 100644 --- a/ormar/models/mixins/merge_mixin.py +++ b/ormar/models/mixins/merge_mixin.py @@ -1,5 +1,7 @@ from typing import TYPE_CHECKING, Optional, cast +import ormar_rust_utils + import ormar from ormar.models.excludable import skip_ellipsis from ormar.queryset.utils import translate_list_to_dict @@ -71,15 +73,22 @@ def merge_instances_list( :rtype: list["Model"] """ merged_rows: list["Model"] = [] - grouped_instances: dict = {} - - for model in result_rows: - grouped_instances.setdefault(model.pk, []).append(model) - for group in grouped_instances.values(): - model = cls._recursive_add(group)[0] - object.__setattr__(model, "__ormar_excludable__", excludable) - merged_rows.append(model) + if result_rows: + pks = [model.pk for model in result_rows] + index_groups = ormar_rust_utils.group_by_pk(pks) + for group_indices in index_groups: + # Single-row groups are the common case for queries with no + # parent duplication (``Model.objects.all()`` and similar); + # skip the wrapper list and the no-op ``_recursive_add`` call. + if len(group_indices) == 1: + model = result_rows[group_indices[0]] + else: + model = cls._recursive_add([result_rows[i] for i in group_indices])[ + 0 + ] + object.__setattr__(model, "__ormar_excludable__", excludable) + merged_rows.append(model) return merged_rows @@ -166,19 +175,26 @@ def _merge_items_lists( :return: merged list of models :rtype: list[Model] """ - value_to_set = [x for x in other_value] - for cur_field in current_field: - if cur_field in other_value: - old_value = next((x for x in other_value if x == cur_field), None) - new_val = cls.merge_two_instances( - cur_field, - cast("Model", old_value), - relation_map=cast( - Optional[dict], - skip_ellipsis(relation_map, field_name, default=dict()), - ), + current_pks = [getattr(m, "pk", None) for m in current_field] + other_pks = [getattr(m, "pk", None) for m in other_value] + plan = ormar_rust_utils.plan_merge_items_lists(current_pks, other_pks) + value_to_set = list(other_value) + nested_relation_map = cast( + Optional[dict], + skip_ellipsis(relation_map, field_name, default=dict()), + ) + for cur_idx, other_idx in plan: + cur_item = current_field[cur_idx] + if other_idx is not None: + # ``other_idx`` is the destination slot the Rust planner + # already identified — write the merged instance there in + # place rather than rebuilding ``value_to_set`` with a pk + # filter (which was O(N) per match). + value_to_set[other_idx] = cls.merge_two_instances( + cur_item, + cast("Model", other_value[other_idx]), + relation_map=nested_relation_map, ) - value_to_set = [x for x in value_to_set if x != cur_field] + [new_val] else: - value_to_set.append(cur_field) + value_to_set.append(cur_item) return value_to_set diff --git a/ormar/models/mixins/relation_mixin.py b/ormar/models/mixins/relation_mixin.py index 3e935bd8c..ecd6381a7 100644 --- a/ormar/models/mixins/relation_mixin.py +++ b/ormar/models/mixins/relation_mixin.py @@ -27,13 +27,17 @@ def extract_db_own_fields(cls) -> set: :return: set of model fields with relation fields excluded :rtype: set """ - related_names = cls.extract_related_names() - self_fields = { - name - for name in cls.ormar_config.model_fields.keys() - if name not in related_names - } - return self_fields + try: + return cls._db_own_fields # type: ignore[attr-defined] + except AttributeError: + related_names = cls.extract_related_names() + self_fields = { + name + for name in cls.ormar_config.model_fields.keys() + if name not in related_names + } + cls._db_own_fields = self_fields # type: ignore[attr-defined] + return self_fields @classmethod def extract_related_fields(cls) -> list["ForeignKeyField"]: diff --git a/ormar/models/mixins/save_mixin.py b/ormar/models/mixins/save_mixin.py index 869136fd0..8466d3006 100644 --- a/ormar/models/mixins/save_mixin.py +++ b/ormar/models/mixins/save_mixin.py @@ -92,7 +92,11 @@ def _remove_not_ormar_fields(cls, new_kwargs: dict) -> dict: :return: dictionary of model that is about to be saved :rtype: dict[str, str] """ - ormar_fields = {k for k, v in cls.ormar_config.model_fields.items()} + try: + ormar_fields = cls._ormar_fields_set # type: ignore[attr-defined] + except AttributeError: + ormar_fields = set(cls.ormar_config.model_fields.keys()) + cls._ormar_fields_set = ormar_fields # type: ignore[attr-defined] new_kwargs = {k: v for k, v in new_kwargs.items() if k in ormar_fields} return new_kwargs diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index 9526805f3..41ddb2074 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional, Union, cast try: @@ -14,6 +15,30 @@ from ormar.models import Model +@dataclass(frozen=True) +class RowExtractionPlan: + """ + Precomputed per-``(model_cls, table_prefix, excludable)`` view of the work + that ``from_row`` used to redo for every row: which columns to read from + the SA row (already prefixed and filtered), which field names to nullify + after construction, and the model's pk field name. + + :ivar column_mappings: ordered ``(prefixed_db_column, field_name)`` pairs + the row reader iterates over to populate the model dict + :ivar excluded_field_names: field names to nullify via + ``_construct_with_excluded`` after pydantic validation + :ivar pk_field_name: cached ``ormar_config.pkname`` so the row reader can + test for a populated pk without a per-row attribute lookup + """ + + column_mappings: tuple[tuple[str, str], ...] + excluded_field_names: frozenset[str] + pk_field_name: str + + +PlanCache = dict[tuple[type, str, int], RowExtractionPlan] + + class ModelRow(NewBaseModel): @classmethod def from_row( # noqa: CFQ002 @@ -27,6 +52,7 @@ def from_row( # noqa: CFQ002 current_relation_str: str = "", proxy_source_model: Optional[type["Model"]] = None, used_prefixes: Optional[list[str]] = None, + plan_cache: Optional[PlanCache] = None, ) -> Optional["Model"]: """ Model method to convert raw sql row from database into ormar.Model instance. @@ -90,17 +116,17 @@ def from_row( # noqa: CFQ002 proxy_source_model=proxy_source_model, # type: ignore table_prefix=table_prefix, used_prefixes=used_prefixes, + plan_cache=plan_cache, ) - item = cls.extract_prefixed_table_columns( - item=item, row=row, table_prefix=table_prefix, excludable=excludable - ) + plan = cls.get_or_build_row_plan(table_prefix, excludable, plan_cache) + item = cls.apply_row_plan(plan, row, item) instance: Optional["Model"] = None - if item.get(cls.ormar_config.pkname, None) is not None: - excluded = cls.get_names_to_exclude( - excludable=excludable, alias=table_prefix + if item.get(plan.pk_field_name, None) is not None: + instance = cast( + "Model", + cls._construct_with_excluded(plan.excluded_field_names, **item), ) - instance = cast("Model", cls._construct_with_excluded(excluded, **item)) instance.set_save_status(True) return instance @@ -154,6 +180,7 @@ def _populate_nested_models_from_row( # noqa: CFQ002 used_prefixes: list[str], current_relation_str: Optional[str] = None, proxy_source_model: Optional[type["Model"]] = None, + plan_cache: Optional[PlanCache] = None, ) -> dict: """ Traverses structure of related models and populates the nested models @@ -208,6 +235,7 @@ def _populate_nested_models_from_row( # noqa: CFQ002 source_model=source_model, proxy_source_model=proxy_source_model, used_prefixes=used_prefixes, + plan_cache=plan_cache, ) item[model_cls.get_column_name_from_alias(related)] = child if ( @@ -222,6 +250,7 @@ def _populate_nested_models_from_row( # noqa: CFQ002 excludable=excludable, child=child, proxy_source_model=proxy_source_model, + plan_cache=plan_cache, ) return item @@ -262,6 +291,7 @@ def _populate_through_instance( # noqa: CFQ002 excludable: ExcludableItems, child: "Model", proxy_source_model: Optional[type["Model"]], + plan_cache: Optional[PlanCache] = None, ) -> None: """ Populates the through model on reverse side of current query. @@ -279,10 +309,16 @@ def _populate_through_instance( # noqa: CFQ002 :type child: "Model" :param proxy_source_model: source model from which querysetproxy is constructed :type proxy_source_model: type["Model"] + :param plan_cache: optional per-queryset plan cache + :type plan_cache: Optional[PlanCache] """ through_name = cls.ormar_config.model_fields[related].through.get_name() through_child = cls._create_through_instance( - row=row, related=related, through_name=through_name, excludable=excludable + row=row, + related=related, + through_name=through_name, + excludable=excludable, + plan_cache=plan_cache, ) if child.__class__ != proxy_source_model: @@ -298,6 +334,7 @@ def _create_through_instance( through_name: str, related: str, excludable: ExcludableItems, + plan_cache: Optional[PlanCache] = None, ) -> "ModelRow": """ Initialize the through model from db row. @@ -311,6 +348,8 @@ def _create_through_instance( :type related: str :param excludable: structure of fields to include and exclude :type excludable: ExcludableItems + :param plan_cache: optional per-queryset plan cache + :type plan_cache: Optional[PlanCache] :return: initialized through model without relation :rtype: "ModelRow" """ @@ -318,63 +357,110 @@ def _create_through_instance( table_prefix = cls.ormar_config.alias_manager.resolve_relation_alias( from_model=cls, relation_name=related ) - # remove relations on through field + # remove relations on through field — must happen before the plan is + # built so the plan reflects the through-model's full exclude set model_excludable = excludable.get(model_cls=model_cls, alias=table_prefix) model_excludable.set_values( value=model_cls.extract_related_names(), slot="exclude" ) - child_dict = model_cls.extract_prefixed_table_columns( - item={}, row=row, excludable=excludable, table_prefix=table_prefix - ) - excluded = model_cls.get_names_to_exclude( - excludable=excludable, alias=table_prefix + plan = model_cls.get_or_build_row_plan(table_prefix, excludable, plan_cache) + child_dict = model_cls.apply_row_plan(plan, row, {}) + child = model_cls._construct_with_excluded( # type: ignore + plan.excluded_field_names, **child_dict ) - child = model_cls._construct_with_excluded(excluded, **child_dict) # type: ignore return child @classmethod - def extract_prefixed_table_columns( + def build_row_extraction_plan( cls, - item: dict, - row: ResultProxy, table_prefix: str, excludable: ExcludableItems, - ) -> dict: + ) -> RowExtractionPlan: """ - Extracts own fields from raw sql result, using a given prefix. - Prefix changes depending on the table's position in a join. + Compute the per-row extraction plan for a ``(cls, table_prefix, + excludable)`` triple — the work that previously ran inside + ``extract_prefixed_table_columns`` for every row. - If the table is a main table, there is no prefix. - All joined tables have prefixes to allow duplicate column names, - as well as duplicated joins to the same table from multiple different tables. - - Extracted fields populates the related dict later used to construct a Model. + :param table_prefix: prefix of the table from AliasManager + :type table_prefix: str + :param excludable: structure of fields to include and exclude + :type excludable: ExcludableItems + :return: cacheable plan for fast per-row extraction + :rtype: RowExtractionPlan + """ + selected_columns = set( + cls.own_table_columns( + model=cls, excludable=excludable, alias=table_prefix, use_alias=False + ) + ) + column_prefix = table_prefix + "_" if table_prefix else "" + column_pairs = cls._get_table_column_pairs(cls) + mappings = tuple( + (f"{column_prefix}{col_name}", field_name) + for col_name, field_name in column_pairs + if field_name in selected_columns + ) + excluded = frozenset( + cls.get_names_to_exclude(excludable=excludable, alias=table_prefix) + ) + return RowExtractionPlan( + column_mappings=mappings, + excluded_field_names=excluded, + pk_field_name=cls.ormar_config.pkname, + ) - Used in Model.from_row and PrefetchQuery._populate_rows methods. + @classmethod + def get_or_build_row_plan( + cls, + table_prefix: str, + excludable: ExcludableItems, + plan_cache: Optional[PlanCache], + ) -> RowExtractionPlan: + """ + Return a cached plan for the given key, or build and cache one. When + ``plan_cache`` is ``None`` (legacy / external caller) the plan is + built fresh on every call so behavior matches the pre-cache shape. + :param table_prefix: prefix of the table from AliasManager + :type table_prefix: str :param excludable: structure of fields to include and exclude :type excludable: ExcludableItems - :param item: dictionary of already populated nested models, otherwise empty dict - :type item: dict + :param plan_cache: per-queryset cache keyed by + ``(cls, table_prefix, id(excludable))``; ``None`` to bypass + :type plan_cache: Optional[PlanCache] + :return: extraction plan for this row position + :rtype: RowExtractionPlan + """ + if plan_cache is None: + return cls.build_row_extraction_plan(table_prefix, excludable) + key = (cls, table_prefix, id(excludable)) + plan = plan_cache.get(key) + if plan is None: + plan = cls.build_row_extraction_plan(table_prefix, excludable) + plan_cache[key] = plan + return plan + + @staticmethod + def apply_row_plan( + plan: RowExtractionPlan, + row: ResultProxy, + item: dict, + ) -> dict: + """ + Populate ``item`` from ``row`` using ``plan.column_mappings``. Skips + keys already present so a partially populated dict (e.g. from + ``_populate_nested_models_from_row``) is not overwritten. + + :param plan: precomputed extraction plan + :type plan: RowExtractionPlan :param row: raw result row from the database :type row: sqlalchemy.engine.result.ResultProxy - :param table_prefix: prefix of the table from AliasManager - each pair of tables have own prefix (two of them depending on direction) - - used in joins to allow multiple joins to the same table. - :type table_prefix: str - :return: dictionary with keys corresponding to model fields names - and values are database values + :param item: dict to populate in place + :type item: dict + :return: ``item`` (returned for chaining symmetry with the legacy API) :rtype: dict """ - selected_columns = cls.own_table_columns( - model=cls, excludable=excludable, alias=table_prefix, use_alias=False - ) - - column_prefix = table_prefix + "_" if table_prefix else "" - for column in cls.ormar_config.table.columns: - alias = cls.get_column_name_from_alias(column.name) - if alias not in item and alias in selected_columns: - prefixed_name = f"{column_prefix}{column.name}" - item[alias] = row[prefixed_name] - + for prefixed_name, field_name in plan.column_mappings: + if field_name not in item: + item[field_name] = row[prefixed_name] return item diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index eac14a1d9..c59be7db2 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -114,6 +114,10 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass _json_fields: set _bytes_fields: set _onupdate_fields: set + _pydantic_field_names: Optional[frozenset[str]] + _extra_is_ignore: Optional[bool] + _allowed_kwarg_names: Optional[frozenset[str]] + _relation_field_names: Optional[frozenset[str]] ormar_config: OrmarConfig # noinspection PyMissingConstructor @@ -223,14 +227,14 @@ def _register_related_models( @classmethod def _construct_with_excluded( - cls, excluded: set[str], **kwargs: Any + cls, excluded: AbstractSet[str], **kwargs: Any ) -> typing_extensions.Self: """ Constructs model instance and nullifies excluded fields post-construction. Used when loading partial results from the database. - :param excluded: set of field names to nullify after construction - :type excluded: set[str] + :param excluded: collection of field names to nullify after construction + :type excluded: AbstractSet[str] :param kwargs: field values for the model :type kwargs: Any :return: constructed model instance @@ -320,7 +324,11 @@ def _update_relation_cache(self, prev_hash: int, new_hash: int) -> None: def _update_cache(relations: list[Relation], recurse: bool = True) -> None: for relation in relations: - relation_proxy = relation.get() + # Read ``related_models`` directly (rather than calling + # ``relation.get()``) so an un-materialized reverse/m2m + # proxy stays un-materialized — there is nothing in an + # empty proxy to migrate hashes for. + relation_proxy = relation.related_models if hasattr(relation_proxy, "update_cache"): relation_proxy.update_cache(prev_hash, new_hash) # type: ignore @@ -374,72 +382,91 @@ def _process_kwargs(self, kwargs: dict) -> tuple[dict, dict]: # noqa: CCR001 :return: modified kwargs :rtype: tuple[dict, dict] """ - property_fields = self.ormar_config.property_fields - model_fields = self.ormar_config.model_fields - pydantic_fields = set(self.__class__.model_fields.keys()) + cls = type(self) + config = cls.ormar_config + model_fields = config.model_fields + + pydantic_fields = cls._pydantic_field_names + if pydantic_fields is None: + pydantic_fields = frozenset(cls.model_fields.keys()) + cls._pydantic_field_names = pydantic_fields # remove property fields - for prop_filed in property_fields: - kwargs.pop(prop_filed, None) + for prop_field in config.property_fields: + kwargs.pop(prop_field, None) if "pk" in kwargs: - kwargs[self.ormar_config.pkname] = kwargs.pop("pk") + kwargs[config.pkname] = kwargs.pop("pk") # extract through fields - through_tmp_dict = dict() - for field_name in self.extract_through_names(): - through_tmp_dict[field_name] = kwargs.pop(field_name, None) + through_tmp_dict = { + field_name: kwargs.pop(field_name, None) + for field_name in self.extract_through_names() + } - kwargs = self._remove_extra_parameters_if_they_should_be_ignored( - kwargs=kwargs, model_fields=model_fields, pydantic_fields=pydantic_fields - ) - try: - new_kwargs: dict[str, Any] = { - k: self._convert_to_bytes( - k, - self._convert_json( - k, - ( - model_fields[k].expand_relationship( - v, self, to_register=False - ) - if k in model_fields - else (v if k in pydantic_fields else model_fields[k]) - ), - ), - ) - for k, v in kwargs.items() - } - except KeyError as e: - raise ModelError( - f"Unknown field '{e.args[0]}' for model {self.get_name(lower=False)}" + extra_is_ignore = cls._extra_is_ignore + if extra_is_ignore is None: + extra_is_ignore = config.extra == Extra.ignore + cls._extra_is_ignore = extra_is_ignore + + if extra_is_ignore: + allowed = cls._allowed_kwarg_names + if allowed is None: + allowed = frozenset(model_fields.keys()) | pydantic_fields + cls._allowed_kwarg_names = allowed + kwargs = {k: v for k, v in kwargs.items() if k in allowed} + + json_fields = cls._json_fields + bytes_fields = cls._bytes_fields + + # ``relation_field_names`` is the disjoint set of fields that need + # ``expand_relationship``; everything else can skip that call. Cached + # on the class on first access — same pattern as ``_pydantic_field_names``. + relation_field_names = getattr(cls, "_relation_field_names", None) + if relation_field_names is None: + relation_field_names = frozenset( + name for name, f in model_fields.items() if f.is_relation ) + cls._relation_field_names = relation_field_names # type: ignore[attr-defined] + + has_json = bool(json_fields) + has_bytes = bool(bytes_fields) + has_relations = bool(relation_field_names) + + # Validate unknown kwargs up front so the dispatch loop doesn't need + # a per-iteration check. ``extra=ignore`` already filtered above, so + # in that branch no unknowns can remain. + if not extra_is_ignore: + for k in kwargs: + if k not in model_fields and k not in pydantic_fields: + try: + model_fields[k] + except KeyError as e: + raise ModelError( + f"Unknown field '{e.args[0]}' for model " + f"{self.get_name(lower=False)}" + ) + if not has_json and not has_bytes and not has_relations: + # Fast path — plain model with no relations/json/bytes. The + # validation pass above is the only per-key cost; the value + # copy is a single C-level dict construction. + return dict(kwargs), through_tmp_dict + + new_kwargs: dict[str, Any] = {} + for k, v in kwargs.items(): + if k in relation_field_names: + v = model_fields[k].expand_relationship(v, self, to_register=False) + if has_json and k in json_fields: + v = encode_json(v) + if has_bytes and k in bytes_fields and v is not None: + v = decode_bytes( + value=v, + represent_as_string=model_fields[k].represent_as_base64_str, + ) + new_kwargs[k] = v return new_kwargs, through_tmp_dict - def _remove_extra_parameters_if_they_should_be_ignored( - self, kwargs: dict, model_fields: dict, pydantic_fields: set - ) -> dict: - """ - Removes the extra fields from kwargs if they should be ignored. - - :param kwargs: passed arguments - :type kwargs: dict - :param model_fields: dictionary of model fields - :type model_fields: dict - :param pydantic_fields: set of pydantic fields names - :type pydantic_fields: set - :return: dict without extra fields - :rtype: dict - """ - if self.ormar_config.extra == Extra.ignore: - kwargs = { - k: v - for k, v in kwargs.items() - if k in model_fields or k in pydantic_fields - } - return kwargs - def _initialize_internal_attributes(self) -> None: """ Initializes internal attributes during __init__() @@ -470,18 +497,26 @@ def __eq__(self, other: object) -> bool: return super().__eq__(other) # pragma no cover def __hash__(self) -> int: - if getattr(self, "__cached_hash__", None) is not None: - return self.__cached_hash__ or 0 - - if self.pk is not None: - ret = hash(str(self.pk) + self.__class__.__name__) + cached = getattr(self, "__cached_hash__", None) + if cached is not None: + return cached + + pk = self.pk + cls = type(self) + if pk is not None: + # ``type(self)`` hashes by identity in CPython, so ``hash((pk, cls))`` + # is uniqueness-equivalent to the original ``str(pk) + cls.__name__`` + # without two string allocations per call. This is the hot path — + # everything that goes through ``_relation_cache`` is keyed on + # saved-pk Models. + ret = hash((pk, cls)) else: - vals = { - k: v - for k, v in self.__dict__.items() - if k not in self.extract_related_names() - } - ret = hash(str(vals) + self.__class__.__name__) + # Unsaved models can hold list/dict values in ``__dict__`` (json + # fields, reverse-relation slots), so we still ``str(vals)`` to + # keep the result hashable. Cold path; not perf-critical. + related = self.extract_related_names() + vals = {k: v for k, v in self.__dict__.items() if k not in related} + ret = hash((str(vals), cls)) object.__setattr__(self, "__cached_hash__", ret) return ret @@ -489,18 +524,23 @@ def __hash__(self) -> int: def __same__(self, other: "NewBaseModel") -> bool: """ Used by __eq__, compares other model to this model. - Compares: - * _orm_ids, - * primary key values if it's set - * dictionary of own fields (excluding relations) + + Saved models (both with pk) compare directly by ``(pk, type)`` to + skip the hash-cache fill on the *other* side. Unsaved/mixed states + fall through to the original hash-equality semantics. + :param other: model to compare to :type other: NewBaseModel :return: result of comparison :rtype: bool """ - if (self.pk is None and other.pk is not None) or ( - self.pk is not None and other.pk is None - ): + if type(self) is not type(other): + return False # pragma: no cover + self_pk = self.pk + other_pk = other.pk + if self_pk is not None and other_pk is not None: + return self_pk == other_pk + if (self_pk is None) != (other_pk is None): return False else: return hash(self) == other.__hash__() @@ -515,10 +555,13 @@ def get_name(cls, lower: bool = True) -> str: :return: name of the model :rtype: str """ - name = cls.__name__ if lower: - name = name.lower() - return name + try: + return cls._lower_name # type: ignore[attr-defined] + except AttributeError: + cls._lower_name = cls.__name__.lower() # type: ignore[attr-defined] + return cls._lower_name # type: ignore[attr-defined] + return cls.__name__ @property def pk_column(self) -> sqlalchemy.Column: @@ -1226,28 +1269,6 @@ def update_from_dict(self, value_dict: builtins.dict) -> "NewBaseModel": setattr(self, key, value) return self - def _convert_to_bytes( - self, column_name: str, value: Any - ) -> Union[str, builtins.dict]: - """ - Converts value to bytes from string - - :param column_name: name of the field - :type column_name: str - :param value: value fo the field - :type value: Any - :return: converted value if needed, else original value - :rtype: Any - """ - if column_name not in self._bytes_fields: - return value - field = self.ormar_config.model_fields[column_name] - if value is not None: - value = decode_bytes( - value=value, represent_as_string=field.represent_as_base64_str - ) - return value - def _convert_bytes_to_str( self, column_name: str, value: Any ) -> Union[str, builtins.dict]: @@ -1272,23 +1293,6 @@ def _convert_bytes_to_str( return base64.b64encode(value).decode() return value - def _convert_json( - self, column_name: str, value: Any - ) -> Union[str, builtins.dict, None]: - """ - Converts value to/from json if needed (for Json columns). - - :param column_name: name of the field - :type column_name: str - :param value: value fo the field - :type value: Any - :return: converted value if needed, else original value - :rtype: Any - """ - if column_name not in self._json_fields: - return value - return encode_json(value) - def _extract_own_model_fields(self) -> builtins.dict: """ Returns a dictionary with field names and values for fields that are not diff --git a/ormar/models/quick_access_views.py b/ormar/models/quick_access_views.py index d073d1fe4..aefa98d44 100644 --- a/ormar/models/quick_access_views.py +++ b/ormar/models/quick_access_views.py @@ -22,7 +22,6 @@ "__private_attributes__", "__same__", "_calculate_keys", - "_convert_json", "_extract_db_related_names", "_extract_model_db_fields", "_extract_nested_models", diff --git a/ormar/queryset/queries/prefetch_query.py b/ormar/queryset/queries/prefetch_query.py index 2eb9e9948..ccd308599 100644 --- a/ormar/queryset/queries/prefetch_query.py +++ b/ormar/queryset/queries/prefetch_query.py @@ -3,6 +3,8 @@ from abc import abstractmethod from typing import TYPE_CHECKING, Any, Sequence, Union, cast +import ormar_rust_utils + import ormar # noqa: I100, I202 from ormar.queryset.clause import QueryClause from ormar.queryset.queries.query import Query @@ -15,16 +17,7 @@ logger = logging.getLogger(__name__) - -class UniqueList(list): - """ - Simple subclass of list that prevents the duplicates - Cannot use set as the order is important - """ - - def append(self, item: Any) -> None: - if item not in self: - super().append(item) +UniqueList = ormar_rust_utils.UniqueList class Node(abc.ABC): @@ -384,14 +377,15 @@ def _instantiate_models(self) -> None: fields_to_exclude = self.relation_field.to.get_names_to_exclude( excludable=self.excludable, alias=self.exclude_prefix ) + # ``self.table_prefix`` and ``self.exclude_prefix`` can differ, so the + # plan only amortizes the column mapping work — exclude set is reused + # from above. Build once before the loop. + row_plan = self.relation_field.to.build_row_extraction_plan( + self.table_prefix, self.excludable + ) parsed_rows: dict[tuple, "Model"] = {} for row in self.rows: - item = self.relation_field.to.extract_prefixed_table_columns( - item={}, - row=row, - table_prefix=self.table_prefix, - excludable=self.excludable, - ) + item = self.relation_field.to.apply_row_plan(row_plan, row, {}) hashable_item = self._hash_item(item) instance = parsed_rows.setdefault( hashable_item, @@ -411,15 +405,7 @@ def _hash_item(self, item: Union[dict, list]) -> tuple: :return: tuple out of model dictionary or list :rtype: tuple """ - result = [] - for key, value in ( - sorted(item.items()) if isinstance(item, dict) else enumerate(item) - ): - if isinstance(value, (dict, list)): - value = self._hash_item(value) - result.append((key, value)) - - return tuple(result) + return ormar_rust_utils.hash_item(item) def _group_models_by_relation_key(self) -> None: """ diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 51bba8630..a072f485b 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -170,15 +170,23 @@ async def _prefetch_related_models( ) return await query.prefetch_related(models=models) # type: ignore - async def _process_query_result_rows(self, rows: list) -> list["T"]: + async def _process_query_result_rows( + self, rows: list, plan_cache: Optional[dict] = None + ) -> list["T"]: """ Process database rows and initialize ormar Model from each of the rows. :param rows: list of database rows from query result :type rows: list[sqlalchemy.engine.result.RowProxy] + :param plan_cache: optional row-extraction plan cache; ``iterate`` + passes a single dict shared across all chunks so amortization + survives the per-chunk ``_process_query_result_rows`` boundary + :type plan_cache: Optional[dict] :return: list of models :rtype: list[Model] """ + if plan_cache is None: + plan_cache = {} result_rows = [] for i, row in enumerate(rows): result_rows.append( @@ -188,6 +196,7 @@ async def _process_query_result_rows(self, rows: list) -> list["T"]: excludable=self._excludable, source_model=self.model, proxy_source_model=self.proxy_source_model, + plan_cache=plan_cache, ) ) if i % 100 == 99: # pragma: no cover @@ -1267,6 +1276,9 @@ async def iterate( # noqa: A003 rows: list = [] last_primary_key = None pk_alias = self.model.get_column_alias(self.model_config.pkname) + # Single shared cache across all yielded chunks so 1-row chunks + # (the common iterate case) still amortize the plan build. + plan_cache: dict = {} # Server-side cursor (asyncpg/aiomysql) requires an open transaction, # which AUTOCOMMIT does not provide. @@ -1280,12 +1292,12 @@ async def iterate( # noqa: A003 rows.append(row) continue - yield (await self._process_query_result_rows(rows))[0] + yield (await self._process_query_result_rows(rows, plan_cache))[0] last_primary_key = current_primary_key rows = [row] if rows: - yield (await self._process_query_result_rows(rows))[0] + yield (await self._process_query_result_rows(rows, plan_cache))[0] async def create(self, **kwargs: Any) -> "T": """ diff --git a/ormar/queryset/utils.py b/ormar/queryset/utils.py index 7e34c25f8..e60f1ce5e 100644 --- a/ormar/queryset/utils.py +++ b/ormar/queryset/utils.py @@ -3,6 +3,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Iterable, Optional, Union +import ormar_rust_utils + from ormar.exceptions import QueryDefinitionError if TYPE_CHECKING: # pragma no cover @@ -310,30 +312,6 @@ def _reverse_range(start: int, stop: int) -> SliceBounds: return SliceBounds(limit=stop - start, offset=-stop, reverse=True) -def check_node_not_dict_or_not_last_node( - part: str, is_last: bool, current_level: Any -) -> bool: - """ - Checks if given name is not present in the current level of the structure. - Checks if given name is not the last name in the split list of parts. - Checks if the given name in current level is not a dictionary. - - All those checks verify if there is a need for deeper traversal. - - :param part: - :type part: str - :param is_last: flag to check if last element - :type is_last: bool - :param current_level: current level of the traversed structure - :type current_level: Any - :return: result of the check - :rtype: bool - """ - return (part not in current_level and not is_last) or ( - part in current_level and not isinstance(current_level[part], dict) - ) - - def translate_list_to_dict( # noqa: CCR001 list_to_trans: Union[list, set], default: Any = ... ) -> dict: @@ -354,21 +332,7 @@ def translate_list_to_dict( # noqa: CCR001 :return: converted to dictionary input list :rtype: dict """ - new_dict: dict = dict() - for path in list_to_trans: - current_level = new_dict - parts = path.split("__") - def_val: Any = copy.deepcopy(default) - for ind, part in enumerate(parts): - is_last = ind == len(parts) - 1 - if check_node_not_dict_or_not_last_node( - part=part, is_last=is_last, current_level=current_level - ): - current_level[part] = dict() - elif part not in current_level: - current_level[part] = def_val - current_level = current_level[part] - return new_dict + return ormar_rust_utils.translate_list_to_dict(list(list_to_trans), default) def convert_set_to_required_dict(set_to_convert: set) -> dict: diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index 81f8c2105..7d418ef04 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -64,7 +64,7 @@ def queryset(self) -> "QuerySet[T]": :rtype: QuerySet """ if not self._queryset: - raise AttributeError + raise AttributeError # pragma: no cover return self._queryset @queryset.setter diff --git a/ormar/relations/relation.py b/ormar/relations/relation.py index 20e5a8841..462708ae8 100644 --- a/ormar/relations/relation.py +++ b/ormar/relations/relation.py @@ -64,11 +64,30 @@ def __init__( self.to: type["T"] = to self._through = through self.field_name: str = field_name - self.related_models: Optional[Union[RelationProxy, "Model"]] = ( - RelationProxy(relation=self, type_=type_, to=to, field_name=field_name) - if type_ in (RelationType.REVERSE, RelationType.MULTIPLE) - else None - ) + # ``RelationProxy`` is built lazily on the first reverse/m2m use + # (``add`` / ``get`` / ``_clean_related``) to avoid the per-model + # allocation when the relation is never read. + self.related_models: Optional[Union[RelationProxy, "Model"]] = None + + def _ensure_proxy(self) -> RelationProxy: + """ + Materialize and cache the ``RelationProxy`` for reverse/m2m relations + on first use. Safe to call multiple times — subsequent calls return + the cached proxy. + + :return: the relation's ``RelationProxy`` + :rtype: RelationProxy + """ + proxy = self.related_models + if not isinstance(proxy, RelationProxy): + proxy = RelationProxy( + relation=self, + type_=self._type, + to=self.to, + field_name=self.field_name, + ) + self.related_models = proxy + return proxy def clear(self) -> None: if self._type in (RelationType.PRIMARY, RelationType.THROUGH): @@ -94,15 +113,15 @@ def _clean_related(self) -> None: for i, x in enumerate(self.related_models) # type: ignore if i not in self._to_remove ] - self.related_models = RelationProxy( + proxy = RelationProxy( relation=self, type_=self._type, to=self.to, field_name=self.field_name, data_=cleaned_data, ) - relation_name = self.field_name - self._owner.__dict__[relation_name] = cleaned_data + self.related_models = proxy + self._owner.__dict__[self.field_name] = proxy self._to_remove = set() def _find_existing( @@ -138,6 +157,12 @@ def add(self, child: "Model") -> None: Adds child Model to relation, either sets child as related model or adds it to the list in RelationProxy depending on relation type. + For reverse / many-to-many relations the ``RelationProxy`` itself is + stored under ``_owner.__dict__[relation_name]``, so a single O(1) + membership check on the proxy's hash cache covers both the relation + bookkeeping and the pydantic-visible ``__dict__`` slot — no parallel + list, no second linear scan. + :param child: model to add to relation :type child: Model """ @@ -145,23 +170,15 @@ def add(self, child: "Model") -> None: if self._type in (RelationType.PRIMARY, RelationType.THROUGH): self.related_models = child self._owner.__dict__[relation_name] = child - else: - if self._find_existing(child) is None: - self.related_models.append(child) # type: ignore - rel = self._owner.__dict__.get(relation_name, []) - rel = rel or [] - if not isinstance(rel, list): - rel = [rel] - self._populate_owner_side_dict(rel=rel, child=child) - self._owner.__dict__[relation_name] = rel - - def _populate_owner_side_dict(self, rel: list["Model"], child: "Model") -> None: - try: - if child not in rel: - rel.append(child) - except ReferenceError: - rel.clear() - rel.append(child) + return + proxy = self._ensure_proxy() + # ``_find_existing`` is the membership check *plus* a dead-weakref + # probe — when a hash collision lands on a stale entry we need that + # probe to populate ``_to_remove`` so the next ``get()`` triggers + # ``_clean_related``. + if self._find_existing(child) is None: + proxy.append(child) + self._owner.__dict__[relation_name] = proxy def remove(self, child: Union["NewBaseModel", type["NewBaseModel"]]) -> None: """ @@ -180,17 +197,25 @@ def remove(self, child: Union["NewBaseModel", type["NewBaseModel"]]) -> None: position = self._find_existing(child) if position is not None: self.related_models.pop(position) # type: ignore - del self._owner.__dict__[relation_name][position] def get(self) -> Optional[Union[list["Model"], "Model"]]: """ Return the related model or models from RelationProxy. + For reverse / many-to-many relations the ``RelationProxy`` is + materialized on first read so callers always see a list-like + return value (even when no children have been registered yet). + :return: related model/models if set :rtype: Optional[Union[list[Model], Model]] """ if self._to_remove: self._clean_related() + if self.related_models is None and self._type in ( + RelationType.REVERSE, + RelationType.MULTIPLE, + ): + return self._ensure_proxy() return self.related_models def __repr__(self) -> str: # pragma no cover diff --git a/ormar/relations/relation_manager.py b/ormar/relations/relation_manager.py index 2f138a0ae..06f057388 100644 --- a/ormar/relations/relation_manager.py +++ b/ormar/relations/relation_manager.py @@ -21,10 +21,15 @@ def __init__( ) -> None: self.owner = proxy(owner) self._related_fields = related_fields or [] - self._related_names = [field.name for field in self._related_fields] + # ``_field_map`` lets ``_get`` build a ``Relation`` lazily by name. + # Holding only the field reference (not a constructed Relation) is + # what skips the per-FK Relation/RelationProxy/QuerysetProxy + # allocation tree on every ``Model.__init__``. + self._field_map: dict[str, "ForeignKeyField"] = { + field.name: field for field in self._related_fields + } + self._related_names = list(self._field_map) self._relations: dict[str, Relation] = dict() - for field in self._related_fields: - self._add_relation(field) def __contains__(self, item: str) -> bool: """ @@ -51,7 +56,7 @@ def get(self, name: str) -> Optional[Union["Model", Sequence["Model"]]]: :return: related model or list of related models if set :rtype: Optional[Union[Model, list[Model]] """ - relation = self._relations.get(name, None) + relation = self._get(name) if relation is not None: return relation.get() return None # pragma nocover @@ -126,17 +131,34 @@ def remove_parent( def _get(self, name: str) -> Optional[Relation]: """ - Returns the actual relation and not the related model(s). + Return the ``Relation`` for ``name``, building it on first access. + + Relations are constructed lazily so that ``Model.__init__`` does + not allocate a ``Relation`` (and, transitively, ``RelationProxy`` / + ``QuerysetProxy``) for every declared FK on every instance — most + of which are never read on row-materialization paths. :param name: name of the relation :type name: str - :return: Relation instance + :return: existing or freshly constructed Relation, or None if the + name does not correspond to a declared relation :rtype: ormar.relations.relation.Relation """ - relation = self._relations.get(name, None) + relation = self._relations.get(name) if relation is not None: return relation - return None + field = self._field_map.get(name) + if field is None: + return None + relation = Relation( + manager=self, + type_=self._get_relation_type(field), + field_name=field.name, + to=field.to, + through=getattr(field, "through", None), + ) + self._relations[name] = relation + return relation def _get_relation_type(self, field: "BaseField") -> RelationType: """ @@ -152,19 +174,3 @@ def _get_relation_type(self, field: "BaseField") -> RelationType: if field.is_through: return RelationType.THROUGH return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE - - def _add_relation(self, field: "BaseField") -> None: - """ - Registers relation in the manager. - Adds Relation instance under field.name. - - :param field: field with relation declaration - :type field: BaseField - """ - self._relations[field.name] = Relation( - manager=self, - type_=self._get_relation_type(field), - field_name=field.name, - to=field.to, - through=getattr(field, "through", None), - ) diff --git a/ormar/relations/relation_proxy.py b/ormar/relations/relation_proxy.py index 94f7c7e34..342a94ea3 100644 --- a/ormar/relations/relation_proxy.py +++ b/ormar/relations/relation_proxy.py @@ -32,9 +32,8 @@ def __init__( self.type_: "RelationType" = type_ self.field_name = field_name self._owner: "Model" = self.relation.manager.owner - self.queryset_proxy: QuerysetProxy[T] = QuerysetProxy[T]( - relation=self.relation, to=to, type_=type_ - ) + self._to: type["T"] = to + self._queryset_proxy: Optional[QuerysetProxy[T]] = None self._related_field_name: Optional[str] = None self._relation_cache: dict[int, int] = {} @@ -51,6 +50,25 @@ def __init__( pass super().__init__(validated_data or ()) + @property + def queryset_proxy(self) -> "QuerysetProxy[T]": + """ + Builds the underlying ``QuerysetProxy`` on first access. Most + ``RelationProxy`` instances are constructed during row materialization + and never have any queryset method invoked on them, so deferring this + allocation skips a non-trivial dict/setattr pair per relation. + + :return: lazily constructed (and cached) QuerysetProxy + :rtype: QuerysetProxy + """ + proxy = self._queryset_proxy + if proxy is None: + proxy = QuerysetProxy[T]( + relation=self.relation, to=self._to, type_=self.type_ + ) + self._queryset_proxy = proxy + return proxy + @property def related_field_name(self) -> str: """ @@ -166,20 +184,38 @@ def __contains__(self, item: object) -> bool: except ReferenceError: return False - def __getattribute__(self, item: str) -> Any: + async def count(self, distinct: bool = True) -> int: # type: ignore[override] """ - Since some QuerySetProxy methods overwrite builtin list methods we - catch calls to them and delegate it to QuerySetProxy instead. + Returns count of related models. Delegates to ``QuerysetProxy.count``. - :param item: name of attribute - :type item: str - :return: value of attribute - :rtype: Any + Defined explicitly to shadow ``list.count`` so attribute lookup resolves + to the relation-aware version without going through ``__getattribute__`` + on every attribute access. + + :param distinct: flag if the primary table rows should be distinct + :type distinct: bool + :return: number of related models + :rtype: int """ - if item in ["count", "clear"]: - self._initialize_queryset() - return getattr(self.queryset_proxy, item) - return super().__getattribute__(item) + self._initialize_queryset() + return await self.queryset_proxy.count(distinct=distinct) + + async def clear(self, keep_reversed: bool = True) -> int: # type: ignore[override] + """ + Removes all related models from the relation. Delegates to + ``QuerysetProxy.clear``. + + Defined explicitly to shadow ``list.clear`` so attribute lookup resolves + to the relation-aware version without going through ``__getattribute__`` + on every attribute access. + + :param keep_reversed: keep reversed FK rows in the database + :type keep_reversed: bool + :return: number of removed relation entries + :rtype: int + """ + self._initialize_queryset() + return await self.queryset_proxy.clear(keep_reversed=keep_reversed) def __getattr__(self, item: str) -> Any: """ @@ -206,14 +242,15 @@ def _initialize_queryset(self) -> None: def _check_if_queryset_is_initialized(self) -> bool: """ - Checks if the QuerySetProxy is already set and ready. + Checks if the QuerySetProxy is already set and ready. Reads the + backing ``_queryset_proxy`` slot directly so the check itself does + not force lazy construction of the proxy. + :return: result of the check :rtype: bool """ - return ( - hasattr(self.queryset_proxy, "queryset") - and self.queryset_proxy.queryset is not None - ) + proxy = self._queryset_proxy + return proxy is not None and proxy._queryset is not None def _check_if_model_saved(self) -> None: """ diff --git a/ormar/utils/__init__.py b/ormar/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poetry.lock b/poetry.lock index 921a616cc..3d00c02d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. [[package]] name = "aiomysql" @@ -7,7 +7,7 @@ description = "MySQL driver for asyncio." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"mysql\"" +markers = "extra == \"mysql\" or extra == \"all\"" files = [ {file = "aiomysql-0.3.2-py3-none-any.whl", hash = "sha256:c82c5ba04137d7afd5c693a258bea8ead2aad77101668044143a991e04632eb2"}, {file = "aiomysql-0.3.2.tar.gz", hash = "sha256:72d15ef5cfc34c03468eb41e1b90adb9fd9347b0b589114bd23ead569a02ac1a"}, @@ -47,7 +47,7 @@ description = "asyncio bridge to the standard sqlite3 module" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"sqlite\"" +markers = "extra == \"sqlite\" or extra == \"all\"" files = [ {file = "aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb"}, {file = "aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650"}, @@ -167,7 +167,7 @@ description = "Timeout context manager for asyncio programs" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"aiopg\" or extra == \"all\" or (extra == \"all\" or extra == \"postgres\" or extra == \"postgresql\" or extra == \"aiopg\") and python_version < \"3.11.0\"" +markers = "extra == \"aiopg\" or extra == \"all\" or (extra == \"postgresql\" or extra == \"postgres\" or extra == \"all\" or extra == \"aiopg\") and python_version < \"3.11.0\"" files = [ {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, @@ -180,7 +180,7 @@ description = "An asyncio PostgreSQL driver" optional = true python-versions = ">=3.9.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\" or extra == \"postgresql\"" +markers = "extra == \"postgresql\" or extra == \"postgres\" or extra == \"all\"" files = [ {file = "asyncpg-0.31.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:831712dd3cf117eec68575a9b50da711893fd63ebe277fc155ecae1c6c9f0f61"}, {file = "asyncpg-0.31.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0b17c89312c2f4ccea222a3a6571f7df65d4ba2c0e803339bfc7bed46a96d3be"}, @@ -400,7 +400,7 @@ files = [ {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] -markers = {main = "(extra == \"all\" or extra == \"crypto\") and platform_python_implementation != \"PyPy\""} +markers = {main = "(extra == \"crypto\" or extra == \"all\") and platform_python_implementation != \"PyPy\""} [package.dependencies] pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} @@ -690,7 +690,7 @@ description = "cryptography is a package which provides cryptographic recipes an optional = true python-versions = "!=3.9.0,!=3.9.1,>=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"crypto\"" +markers = "extra == \"crypto\" or extra == \"all\"" files = [ {file = "cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6"}, {file = "cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c"}, @@ -1716,7 +1716,7 @@ description = "Fast, correct Python JSON library supporting dataclasses, datetim optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"all\" or extra == \"orjson\"" +markers = "extra == \"orjson\" or extra == \"all\"" files = [ {file = "orjson-3.11.9-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:135869ef917b8704ea0a94e01620e0c05021c15c52036e4663baffe75e72f8ce"}, {file = "orjson-3.11.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:115ab5f5f4a0f203cc2a5f0fb09aee503a3f771aa08392949ab5ca230c4fbdbd"}, @@ -1794,6 +1794,36 @@ files = [ {file = "orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f"}, ] +[[package]] +name = "ormar-utils" +version = "0.1.1" +description = "Rust-accelerated utility functions for the ormar ORM" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "ormar_utils-0.1.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:e1abb5494942a937c317aac909979c422aac745c58121042216ba4163f22ef47"}, + {file = "ormar_utils-0.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1f80e577d840ae0f2e6455b84e6e6e6351a776a514af8b6ff1f809f24a71f432"}, + {file = "ormar_utils-0.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:522f6fe1f68f4cd90a77a0851e2bcffbd7d473cb5eb492f53a79ecd249b4a95a"}, + {file = "ormar_utils-0.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77bcfffbef6f81a61192290834b2b80fbfeaa64242a8fbcdefe00051b9f5e70d"}, + {file = "ormar_utils-0.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:3e316b727853f0d722663e8af74aca8f9572f8340a76694ebd8a377fc6d6bfbe"}, + {file = "ormar_utils-0.1.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a5468770de8d01b36702fe96aa553efb30da46313304b97d113871519d4fec5b"}, + {file = "ormar_utils-0.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:047f5e113e3adb68e4dc7d3915d999ed44a3e6914a5a0a8162cfc0f1bcc24a24"}, + {file = "ormar_utils-0.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b510c48557a1809e53a851b7acee4bac8fb7adac12ce738d3911b6ed85dc7bb"}, + {file = "ormar_utils-0.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:530c46dcfcfb26067abd6e148b2c2337cfcefa1945cbffcae8a8b73ee816a101"}, + {file = "ormar_utils-0.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:d1cec4877a86d7f654d795f3f49bb4f8d81843032dc05f5a741ce471672f5c5d"}, + {file = "ormar_utils-0.1.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:85e83277179776bd8d4915ce4a5e30e68054be22f3d4e4428e2f0a5515b113e1"}, + {file = "ormar_utils-0.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2736d32eeeb403f7883a4ffcc67b716e492ef38940d9ca42b9be7f1c958a458a"}, + {file = "ormar_utils-0.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b69e0a1903c1eb8c63f284eec96aa550e44a417d79e673254b83d73b290c4c60"}, + {file = "ormar_utils-0.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da04622f42cc33dd88a94f21dda0f04ff7ff0773e3e3a176ef8d9bad5e60a1c1"}, + {file = "ormar_utils-0.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:a803c9f3b314eb54220bace76f7c49d091f305bba2fa591c19eead33ef5b1741"}, + {file = "ormar_utils-0.1.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f8635e83857b85e2521db8cf55c1c659169912713632b9bc8f7cb617df3235df"}, + {file = "ormar_utils-0.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4892b801969cbb402748a12920b8c285707fdb2540fe423b6c96bcbd6004aa98"}, + {file = "ormar_utils-0.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fae770725f88168f8dcecfafaca27375397a62a97776c2818b5a99c6d4baeb8d"}, + {file = "ormar_utils-0.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5bd489da692e9b94f27a577acf30465b30e94932f01f2745830cb94b1b413ce"}, + {file = "ormar_utils-0.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:405ed56e66cff421f293509d3e0a86af7da88663420a2d3ea5719e755421bb2a"}, +] + [[package]] name = "packaging" version = "24.2" @@ -1928,7 +1958,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"aiopg\" or extra == \"all\" or extra == \"postgres\" or extra == \"postgresql\"" +markers = "extra == \"aiopg\" or extra == \"all\" or extra == \"postgresql\" or extra == \"postgres\"" files = [ {file = "psycopg2_binary-2.9.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9b818ceff717f98851a64bffd4c5eb5b3059ae280276dcecc52ac658dcf006a4"}, {file = "psycopg2_binary-2.9.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2fa0d7caca8635c56e373055094eeda3208d901d55dd0ff5abc1d4e47f82b56"}, @@ -2022,7 +2052,7 @@ files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] -markers = {main = "(extra == \"all\" or extra == \"crypto\") and platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"", dev = "implementation_name != \"PyPy\""} +markers = {main = "(extra == \"crypto\" or extra == \"all\") and platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"", dev = "implementation_name != \"PyPy\""} [[package]] name = "pydantic" @@ -2248,7 +2278,7 @@ description = "Pure Python MySQL Driver" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"mysql\"" +markers = "extra == \"mysql\" or extra == \"all\"" files = [ {file = "pymysql-1.1.3-py3-none-any.whl", hash = "sha256:8164ba62c552f6105f3b11753352d0f16b90d1703ba67d81923d5a8a5d1c5289"}, {file = "pymysql-1.1.3.tar.gz", hash = "sha256:e70ebf2047a4edf6138cf79c68ad418ef620af65900aa585c5e8bfc95044d43a"}, @@ -3110,4 +3140,4 @@ sqlite = ["aiosqlite"] [metadata] lock-version = "2.1" python-versions = "^3.10.0" -content-hash = "35599fdafc67c9ce656613453a927f5e7bae92c3878cafcc4e585999b4101571" +content-hash = "24c5baf41b4913764cb9686979371e82d0dc18f3e4cff5f9da27e7648a4a179c" diff --git a/pyproject.toml b/pyproject.toml index 8b217cb67..3031a6a24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ asyncpg = { version = ">=0.28,<0.32", optional = true } psycopg2-binary = { version = "^2.9.1", optional = true } mysqlclient = { version = "^2.1.0", optional = true } PyMySQL = { version = "^1.1.0", optional = true } +ormar-utils = ">=0.1.1" [tool.poetry.dependencies.orjson] @@ -157,7 +158,7 @@ module = "docs_src.*" ignore_errors = true [[tool.mypy.overrides]] -module = ["sqlalchemy.*", "asyncpg", "nest_asyncio"] +module = ["sqlalchemy.*", "asyncpg", "nest_asyncio", "ormar_rust_utils"] ignore_missing_imports = true [tool.yapf]