optimizations to improve performance, add ormar-utils

.github/workflows/deploy-docs.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1.4
         with:
-          version: 2.1.3
+          version: 2.3.2
           virtualenvs-create: false
       - name: Install dependencies
         run: |

.github/workflows/lint.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: false
 
     - name: Poetry details

.github/workflows/python-publish.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: true
         virtualenvs-in-project: true
 

.github/workflows/test-package.yml

@@ -50,7 +50,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: false
 
     - name: Poetry details

.github/workflows/test_docs.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: false
     - name: Install dependencies
       run: |

.github/workflows/type-check.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: false
 
     - name: Poetry details

benchmarks/test_benchmark_alias_lookup.py

@@ -0,0 +1,68 @@
+"""
+Micro-benchmark for get_column_name_from_alias and related alias functions.
+"""
+
+import pytest
+
+from benchmarks.conftest import Author, Book
+
+pytestmark = pytest.mark.asyncio
+
+
+@pytest.mark.parametrize("num_lookups", [1000, 10000])
+def test_get_column_name_from_alias(benchmark, num_lookups: int) -> None:
+    """Benchmark get_column_name_from_alias - O(n) linear scan per call."""
+    # Get all column aliases for the model
+    aliases = [col.name for col in Author.ormar_config.table.columns]
+
+    def run() -> None:
+        for _ in range(num_lookups):
+            for alias in aliases:
+                Author.get_column_name_from_alias(alias)
+
+    benchmark(run)
+
+
+@pytest.mark.parametrize("num_lookups", [1000, 10000])
+def test_get_column_name_from_alias_book(benchmark, num_lookups: int) -> None:
+    """Benchmark on Book model (more fields including FKs)."""
+    aliases = [col.name for col in Book.ormar_config.table.columns]
+
+    def run() -> None:
+        for _ in range(num_lookups):
+            for alias in aliases:
+                Book.get_column_name_from_alias(alias)
+
+    benchmark(run)
+
+
+@pytest.mark.parametrize("num_lookups", [1000, 10000])
+def test_translate_columns_to_aliases(benchmark, num_lookups: int) -> None:
+    """Benchmark translate_columns_to_aliases - dict key remapping."""
+
+    def run() -> None:
+        for _ in range(num_lookups):
+            kwargs = {"name": "test", "score": 50, "id": 1}
+            Author.translate_columns_to_aliases(kwargs)
+
+    benchmark(run)
+
+
+@pytest.mark.parametrize("num_lookups", [1000, 10000])
+def test_translate_aliases_to_columns(benchmark, num_lookups: int) -> None:
+    """Benchmark translate_aliases_to_columns - reverse remapping."""
+    # Get aliases
+    aliases = {
+        field.get_alias(): "value"
+        for field_name, field in Author.ormar_config.model_fields.items()
+        if field.get_alias()
+    }
+    if not aliases:
+        aliases = {"name": "test", "score": 50, "id": 1}
+
+    def run() -> None:
+        for _ in range(num_lookups):
+            kwargs = dict(aliases)
+            Author.translate_aliases_to_columns(kwargs)
+
+    benchmark(run)

benchmarks/test_benchmark_merge.py

@@ -0,0 +1,118 @@
+"""Benchmark for nested-join row merging in ``_process_query_result_rows``.
+
+The workload below targets the path optimized by the in-place index
+assignment in ``_merge_items_lists``: a parent that fans out across many
+joined rows so the matched branch fires repeatedly during
+``_recursive_add``, with both halves of late-round merges accumulating
+overlapping child PKs.
+
+Shape:
+
+    Project (1) -> Task (N, FK) -> Tag (M, m2m, shared across tasks)
+
+A single ``Project`` produces ``N * M`` result rows on
+``select_related(["tasks", "tasks__tags"]).all()``. The pairwise
+``_recursive_add`` consolidates the duplicates, and in the deeper rounds
+both sides hold overlapping ``Task`` PKs (because every adjacent row
+carries the same task with a different tag); inside each task the tag
+lists also accumulate and overlap across recursion halves.
+"""
+
+import pytest
+
+import ormar
+from benchmarks.conftest import base_ormar_config
+
+pytestmark = pytest.mark.asyncio
+
+
+class BenchTag(ormar.Model):
+    ormar_config = base_ormar_config.copy(tablename="bench_merge_tags")
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=50)
+
+
+class BenchProject(ormar.Model):
+    ormar_config = base_ormar_config.copy(tablename="bench_merge_projects")
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+
+
+class BenchTask(ormar.Model):
+    ormar_config = base_ormar_config.copy(tablename="bench_merge_tasks")
+
+    id: int = ormar.Integer(primary_key=True)
+    project: BenchProject = ormar.ForeignKey(
+        BenchProject, index=True, related_name="tasks"
+    )
+    title: str = ormar.String(max_length=100)
+    tags: list[BenchTag] = ormar.ManyToMany(BenchTag)
+
+
+@pytest.mark.parametrize(
+    ("num_tasks", "tags_per_task"),
+    [(5, 5), (10, 10), (20, 10)],
+)
+async def test_select_related_nested_merge(
+    aio_benchmark, num_tasks: int, tags_per_task: int
+):
+    """Project -> Tasks (FK) -> Tags (m2m) workload.
+
+    Result row count is ``num_tasks * tags_per_task`` for one project; every
+    row materializes a duplicate Project with one Task (with one Tag).
+    The merge path consolidates duplicates pairwise via ``_recursive_add``
+    — covers ``_merge_items_lists`` end-to-end through the full join /
+    materialize / merge pipeline.
+    """
+    project = await BenchProject(name="P").save()
+    tags = [await BenchTag(name=f"t{i}").save() for i in range(tags_per_task)]
+    for i in range(num_tasks):
+        task = await BenchTask(project=project, title=f"T{i}").save()
+        for tag in tags:
+            await task.tags.add(tag)
+
+    @aio_benchmark
+    async def query():
+        return await BenchProject.objects.select_related(["tasks", "tasks__tags"]).all()
+
+    result = query()
+    assert len(result) == 1
+    assert len(result[0].tasks) == num_tasks
+    for task in result[0].tasks:
+        assert len(task.tags) == tags_per_task
+
+
+@pytest.mark.parametrize("list_size", [10, 50, 100])
+def test_merge_items_lists_pk_overlap(benchmark, list_size: int):
+    """Microbenchmark for ``_merge_items_lists`` with full PK overlap.
+
+    Constructs two equally sized lists of saved tasks where every entry
+    in ``current_field`` matches an entry in ``other_value`` by PK. This
+    is the worst case the per-pair list rebuild used to be O(N) on — K
+    matches, each filtering an N-element ``value_to_set``. With
+    ``other_idx`` driving in-place writes the cost drops from O(K·N) to
+    O(K).
+
+    The benchmark calls the merge classmethod directly so the SA query /
+    row materialization overhead is excluded — the signal we want is the
+    inner loop only. Every task is fully populated (no relations to
+    recurse into) so ``merge_two_instances`` is light and the
+    ``_merge_items_lists`` body itself dominates.
+    """
+    project = BenchProject(id=1, name="p")
+    current_field = [
+        BenchTask(id=i, project=project, title=f"T{i}") for i in range(list_size)
+    ]
+    other_value = [
+        BenchTask(id=i, project=project, title=f"T{i}") for i in range(list_size)
+    ]
+
+    benchmark(
+        BenchTask._merge_items_lists,
+        field_name="tasks",
+        current_field=current_field,
+        other_value=other_value,
+        relation_map={"tasks": ...},
+    )

ormar/fields/base.py

@@ -384,7 +384,7 @@ def expand_relationship(
         :return: returns untouched value for normal fields, expands only for relations
         :rtype: Any
         """
-        return value
+        return value  # pragma: no cover
 
     def set_self_reference_flag(self) -> None:
         """

ormar/fields/foreign_key.py

@@ -2,6 +2,7 @@
 import sys
 import uuid
 from dataclasses import dataclass
+from functools import cached_property
 from random import choices
 from typing import TYPE_CHECKING, Any, ForwardRef, Optional, Union, cast, overload
 
@@ -491,28 +492,6 @@ def _extract_model_from_sequence(
             for val in value
         ]
 
-    def _register_existing_model(
-        self, value: "Model", child: "Model", to_register: bool
-    ) -> "Model":
-        """
-        Takes already created instance and registers it for parent.
-        Registration is mutual, so children have also reference to parent.
-
-        Used in reverse FK relations and normal FK for single models.
-
-        :param value: already instantiated Model
-        :type value: Model
-        :param child: child/ related Model
-        :type child: Model
-        :param to_register: flag if the relation should be set in RelationshipManager
-        :type to_register: bool
-        :return: (if needed) registered Model
-        :rtype: Model
-        """
-        if to_register:
-            self.register_relation(model=value, child=child)
-        return value
-
     def _construct_model_from_dict(
         self, value: dict, child: "Model", to_register: bool
     ) -> "Model":
@@ -611,6 +590,23 @@ def has_unresolved_forward_refs(self) -> bool:
         """
         return self.to.__class__ == ForwardRef
 
+    @cached_property
+    def _constructor_dispatch(self) -> dict[str, Any]:
+        """
+        Per-field map from input class name to the constructor handling that
+        shape. Built once at first access — by the time
+        ``expand_relationship`` runs, ``_verify_model_can_be_initialized``
+        has already gated on ``requires_ref_update``, so the bound methods
+        captured here are stable.
+
+        :return: dispatch table for the slow path of ``expand_relationship``
+        :rtype: dict[str, Any]
+        """
+        return {
+            "dict": self._construct_model_from_dict,
+            "list": self._extract_model_from_sequence,
+        }
+
     def expand_relationship(
         self,
         value: Any,
@@ -636,16 +632,17 @@ def expand_relationship(
         """
         if value is None:
             return None if not self.virtual else []
-        constructors = {
-            f"{self.to.__name__}": self._register_existing_model,
-            "dict": self._construct_model_from_dict,
-            "list": self._extract_model_from_sequence,
-        }
-
-        model = constructors.get(  # type: ignore
+        # Fast path: ``value`` is already a Model of ``self.to``. Dominant
+        # case in row materialization and in user kwargs that pass
+        # constructed Models directly. Skips the dispatch table and the
+        # ``_register_existing_model`` indirection entirely.
+        if value.__class__ is self.to:
+            if to_register:
+                self.register_relation(model=value, child=cast("Model", child))
+            return value
+        return self._constructor_dispatch.get(
             value.__class__.__name__, self._construct_model_from_pk
         )(value, child, to_register)
-        return model
 
     def get_relation_name(self) -> str:  # pragma: no cover
         """

ormar/fields/parsers.py

@@ -1,9 +1,9 @@
-import base64
 import datetime
 import decimal
 import uuid
-from typing import Any, Callable, Optional, Union
+from typing import Callable, Optional
 
+import ormar_rust_utils
 import pydantic
 from pydantic_core import SchemaValidator, core_schema
 
@@ -27,43 +27,9 @@ def encode_decimal(value: decimal.Decimal, precision: Optional[int] = None) -> f
     )
 
 
-def encode_bytes(value: Union[str, bytes], represent_as_string: bool = False) -> str:
-    if represent_as_string:
-        value = (
-            value if isinstance(value, str) else base64.b64encode(value).decode("utf-8")
-        )
-    else:
-        value = value if isinstance(value, str) else value.decode("utf-8")
-    return value
-
-
-def decode_bytes(value: str, represent_as_string: bool = False) -> bytes:
-    if represent_as_string:
-        return value if isinstance(value, bytes) else base64.b64decode(value)
-    return value if isinstance(value, bytes) else value.encode("utf-8")
-
-
-def encode_json(value: Any) -> Optional[str]:
-    if isinstance(value, (datetime.date, datetime.datetime, datetime.time)):
-        value = value.isoformat()
-    value = json.dumps(value) if not isinstance(value, str) else re_dump_value(value)
-    value = value.decode("utf-8") if isinstance(value, bytes) else value
-    return value
-
-
-def re_dump_value(value: str) -> Union[str, bytes]:
-    """
-    Re-dumps value due to different string representation in orjson and json
-    :param value: string to re-dump
-    :type value: str
-    :return: re-dumped value
-    :rtype: list[str]
-    """
-    try:
-        result: Union[str, bytes] = json.dumps(json.loads(value))
-    except json.JSONDecodeError:
-        result = value
-    return result
+encode_bytes = ormar_rust_utils.encode_bytes
+decode_bytes = ormar_rust_utils.decode_bytes
+encode_json = ormar_rust_utils.encode_json
 
 
 ENCODERS_MAP: dict[type, Callable] = {