From 6bb7b7e61e626df88cec19ed740b870fa4f196d4 Mon Sep 17 00:00:00 2001 From: Bernhard Bliem Date: Tue, 3 May 2022 17:03:13 +0200 Subject: [PATCH 1/6] Add default_language_field parameter to TranslationField --- modeltrans/fields.py | 55 +++++++++++++++++++++++++++------- modeltrans/forms.py | 10 +++++-- modeltrans/manager.py | 15 ++++++++-- modeltrans/translator.py | 39 ++++++++++++++---------- modeltrans/utils.py | 6 ++-- tests/app/models.py | 24 +++++++++++++++ tests/test_models.py | 63 +++++++++++++++++++++++++++++++++++++-- tests/test_querysets.py | 32 +++++++++++++++++++- tests/test_translating.py | 2 ++ tests/test_utils.py | 48 ++++++++++++++++++++++++++++- 10 files changed, 258 insertions(+), 36 deletions(-) diff --git a/modeltrans/fields.py b/modeltrans/fields.py index 965a7e1..d7f5f3a 100644 --- a/modeltrans/fields.py +++ b/modeltrans/fields.py @@ -1,5 +1,6 @@ from django.core.exceptions import ImproperlyConfigured from django.db.models import F, fields +from django.db.models.expressions import Case, When from django.db.models.functions import Cast, Coalesce from django.utils.translation import gettext @@ -22,10 +23,10 @@ SUPPORTED_FIELDS = (fields.CharField, fields.TextField) -DEFAULT_LANGUAGE = get_default_language() +GLOBAL_DEFAULT_LANGUAGE = get_default_language() -def translated_field_factory(original_field, language=None, *args, **kwargs): +def translated_field_factory(original_field, language=None, default_language_field=None, *args, **kwargs): if not isinstance(original_field, SUPPORTED_FIELDS): raise ImproperlyConfigured( "{} is not supported by django-modeltrans.".format(original_field.__class__.__name__) @@ -36,7 +37,7 @@ class Specific(TranslatedVirtualField, original_field.__class__): Specific.__name__ = "Translated{}".format(original_field.__class__.__name__) - return Specific(original_field, language, *args, **kwargs) + return Specific(original_field, language, *args, default_language_field=default_language_field, **kwargs) class TranslatedVirtualField: @@ -45,18 +46,21 @@ class TranslatedVirtualField: Arguments: original_field: The original field to be translated - language: The language to translate to, or `None` to track the current active Django language. + language: The language to translate to, or `None` to use the default language (see `default_language_field`) + default_language_field: Name of the field that contains the default language for this field, or `None` to track + the current active Django language """ # Implementation inspired by HStoreVirtualMixin from: # https://github.com/djangonauts/django-hstore/blob/master/django_hstore/virtual.py - def __init__(self, original_field, language=None, *args, **kwargs): + def __init__(self, original_field, language=None, *args, default_language_field=None, **kwargs): # TODO: this feels like a big hack. self.__dict__.update(original_field.__dict__) self.original_field = original_field self.language = language + self.default_language_field = default_language_field self.blank = kwargs["blank"] self.null = kwargs["null"] @@ -128,7 +132,8 @@ def __get__(self, instance, instance_type=None): language = self.get_language() original_value = getattr(instance, self.original_name) - if language == DEFAULT_LANGUAGE and original_value: + default_language = self.get_default_language(instance) + if language == default_language and original_value: return original_value # Make sure we test for containment in a dict, not in None @@ -144,7 +149,7 @@ def __get__(self, instance, instance_type=None): # This is the _i18n version of the field, and the current language is not available, # so we walk the fallback chain: for fallback_language in (language,) + self.get_instance_fallback_chain(instance, language): - if fallback_language == DEFAULT_LANGUAGE: + if fallback_language == default_language: if original_value: return original_value else: @@ -163,7 +168,8 @@ def __set__(self, instance, value): language = self.get_language() - if language == DEFAULT_LANGUAGE: + default_language = self.get_default_language(instance) + if language == default_language: setattr(instance, self.original_name, value) else: field_name = build_localized_fieldname(self.original_name, language) @@ -211,7 +217,7 @@ def output_field(self): return Field() def _localized_lookup(self, language, bare_lookup): - if language == DEFAULT_LANGUAGE: + if not self.default_language_field and language == GLOBAL_DEFAULT_LANGUAGE: return bare_lookup.replace(self.name, self.original_name) # When accessing a table directly, the i18_lookup will be just "i18n", while following relations @@ -223,6 +229,15 @@ def _localized_lookup(self, language, bare_lookup): # abuse build_localized_fieldname without language to get "_" field_prefix = build_localized_fieldname(self.original_name, "") return FallbackTransform(field_prefix, language, i18n_lookup) + elif self.default_language_field: + default_value_field = bare_lookup.replace(self.name, self.original_name) + return Case( + When(**{self.default_language_field: language}, then=default_value_field), + default=KeyTextTransform( + build_localized_fieldname(self.original_name, language), i18n_lookup + ), + output_field=self.output_field(), + ) else: return KeyTextTransform( build_localized_fieldname(self.original_name, language), i18n_lookup @@ -233,7 +248,7 @@ def as_expression(self, bare_lookup, fallback=True): Compose an expression to get the value for this virtual field in a query. """ language = self.get_language() - if language == DEFAULT_LANGUAGE: + if not self.default_language_field and language == GLOBAL_DEFAULT_LANGUAGE: return F(self._localized_lookup(language, bare_lookup)) if not fallback: @@ -254,8 +269,22 @@ def as_expression(self, bare_lookup, fallback=True): # and now, add the list of fallback languages to the lookup list for fallback_language in fallback_chain: lookups.append(self._localized_lookup(fallback_language, bare_lookup)) + + if self.default_language_field: + # Add the original field as a fallback (might not be in the fallback chain) + # TBD: This may be a good idea anyway even if self.default_language_field is None. After all, __get__() + # falls back to the original field if all else fails, so it may be surprising that `instance.field_i18n` + # falls back to the original field but `Model.objects.values_list('field_i18n')` does not. Changing this + # might break existing applications though. + lookups.append(bare_lookup.replace(self.name, self.original_name)) + return Coalesce(*lookups, output_field=self.output_field()) + def get_default_language(self, instance): + if not self.default_language_field: + return GLOBAL_DEFAULT_LANGUAGE + return get_instance_field_value(instance, self.default_language_field) + class TranslationField(JSONField): """ @@ -263,6 +292,10 @@ class TranslationField(JSONField): Arguments: fields (iterable): List of model field names to make translatable. + default_language_field (field name): + Field of the model containing the language stored in the original + model fields; use Django main language by default. May contain `__` + as a field name separator to follow foreign keys. required_languages (iterable or dict): List of languages required for the model. If a dict is supplied, the keys must be translated field names with the value containing a list of required languages for that specific field. @@ -282,6 +315,7 @@ class TranslationField(JSONField): def __init__( self, fields=None, + default_language_field=None, required_languages=None, virtual_fields=True, fallback_language_field=None, @@ -289,6 +323,7 @@ def __init__( **kwargs, ): self.fields = fields or () + self.default_language_field = default_language_field self.required_languages = required_languages or () self.virtual_fields = virtual_fields self.fallback_language_field = fallback_language_field diff --git a/modeltrans/forms.py b/modeltrans/forms.py index 43ecc41..87d9171 100644 --- a/modeltrans/forms.py +++ b/modeltrans/forms.py @@ -64,13 +64,16 @@ def __new__(mcs, name, bases, attrs): if model_class: i18n_field = get_i18n_field(model_class) if i18n_field: + has_default_language_field = bool(i18n_field.default_language_field) for original_field_name in i18n_field.fields: # for all translated fields # for all possible system languages for language in languages: + # Ignore the default language suffix (and use the original field) only if the field has no + # specified default_language_field because, if it does, we don't know the default language field_name = build_localized_fieldname( - original_field_name, language, ignore_default=True + original_field_name, language, ignore_default=not has_default_language_field ) # add i18n field if an explicitly chosen field @@ -227,9 +230,12 @@ def get_included_fields(self): """ fields = {} + has_default_language_field = bool(self.model_i18n_field.default_language_field) for original_field in self.i18n_fields: + # Ignore the default language suffix (and use the original field) only if the field has no + # specified default_language_field because, if it does, we don't know the default language fields[original_field] = [ - build_localized_fieldname(original_field, language_code, ignore_default=True) + build_localized_fieldname(original_field, language_code, ignore_default=not has_default_language_field) for language_code in self.language_codes ] fields["__all__"] = list(itertools.chain.from_iterable(fields.values())) diff --git a/modeltrans/manager.py b/modeltrans/manager.py index 48b363a..a89bcf9 100644 --- a/modeltrans/manager.py +++ b/modeltrans/manager.py @@ -6,7 +6,7 @@ from .conf import get_default_language from .fields import TranslatedVirtualField -from .utils import DJANGO_VERSION +from .utils import DJANGO_VERSION, get_instance_field_value def transform_translatable_fields(model, fields): @@ -39,7 +39,16 @@ def transform_translatable_fields(model, fields): if isinstance(field, TranslatedVirtualField): has_translated_fields = True - if field.get_language() == get_default_language(): + if field.default_language_field: + first_part, *path = field.default_language_field.split(LOOKUP_SEP, maxsplit=1) + if path: + assert len(path) == 1 + default_language = get_instance_field_value(fields[first_part], path[0]) + else: + default_language = fields[first_part] + else: + default_language = get_default_language() + if field.get_language() == default_language: if field.original_name in fields: raise ValueError( 'Attempted override of "{}" with "{}". ' @@ -329,7 +338,7 @@ def _values(self, *fields, **expressions): fallback = field.language is None - if field.get_language() == get_default_language(): + if not field.default_language_field and field.get_language() == get_default_language(): original_field = field_name.replace(field.name, field.original_field.name) self.query.add_annotation(Cast(original_field, field.output_field()), field_name) else: diff --git a/modeltrans/translator.py b/modeltrans/translator.py index df2b1b7..3d1e235 100644 --- a/modeltrans/translator.py +++ b/modeltrans/translator.py @@ -61,9 +61,10 @@ def translate_model(Model): validate(Model) add_manager(Model) + default_language_field = get_i18n_field_param(Model, i18n_field, "default_language_field") fields_to_translate = get_i18n_field_param(Model, i18n_field, "fields") required_languages = get_i18n_field_param(Model, i18n_field, "required_languages") - add_virtual_fields(Model, fields_to_translate, required_languages) + add_virtual_fields(Model, default_language_field, fields_to_translate, required_languages) patch_constructor(Model) translate_meta_ordering(Model) @@ -136,7 +137,7 @@ def raise_if_field_exists(Model, field_name): ) -def add_virtual_fields(Model, fields, required_languages): +def add_virtual_fields(Model, default_language_field, fields, required_languages): """ Adds newly created translation fields to the given translation options. """ @@ -154,34 +155,42 @@ def add_virtual_fields(Model, fields, required_languages): # first, add a `_i18n` virtual field to get the currently # active translation for a field field = translated_field_factory( - original_field=original_field, blank=True, null=True, editable=False # disable in admin + original_field=original_field, blank=True, null=True, editable=False, # disable in admin + default_language_field=default_language_field, ) raise_if_field_exists(Model, field.get_field_name()) field.contribute_to_class(Model, field.get_field_name()) - # add a virtual field pointing to the original field with name - # _ - field = translated_field_factory( - original_field=original_field, - language=get_default_language(), - blank=True, - null=True, - editable=False, - ) - raise_if_field_exists(Model, field.get_field_name()) - field.contribute_to_class(Model, field.get_field_name()) + if default_language_field: + # create field for global default language later on + add_field_for_global_default_language = True + else: + # create field for global default language now with different arguments than fields for other languages + add_field_for_global_default_language = False + # add a virtual field pointing to the original field with name + # _ + field = translated_field_factory( + original_field=original_field, + language=get_default_language(), + blank=True, + null=True, + editable=False, + ) + raise_if_field_exists(Model, field.get_field_name()) + field.contribute_to_class(Model, field.get_field_name()) # now, for each language, add a virtual field to get the tranlation for # that specific langauge # _ - for language in get_available_languages(include_default=False): + for language in get_available_languages(include_default=add_field_for_global_default_language): blank_allowed = language not in field_required_languages field = translated_field_factory( original_field=original_field, language=language, blank=blank_allowed, null=blank_allowed, + default_language_field=default_language_field, ) raise_if_field_exists(Model, field.get_field_name()) field.contribute_to_class(Model, field.get_field_name()) diff --git a/modeltrans/utils.py b/modeltrans/utils.py index 40956c2..e6fea98 100644 --- a/modeltrans/utils.py +++ b/modeltrans/utils.py @@ -32,12 +32,14 @@ def split_translated_fieldname(field_name): return (field_name[0:_pos], field_name[_pos + 1 :]) -def build_localized_fieldname(field_name, lang, ignore_default=False): +def build_localized_fieldname(field_name, lang, ignore_default=False, default_language=None): + if default_language is None: + default_language = get_default_language() if lang == "id": # The 2-letter Indonesian language code is problematic with the # current naming scheme as Django foreign keys also add "id" suffix. lang = "ind" - if ignore_default and lang == get_default_language(): + if ignore_default and lang == default_language: return field_name return "{}_{}".format(field_name, lang.replace("-", "_")) diff --git a/tests/app/models.py b/tests/app/models.py index b91753e..9d34c54 100644 --- a/tests/app/models.py +++ b/tests/app/models.py @@ -198,3 +198,27 @@ class ChallengeContent(models.Model): def __str__(self): return self.content_i18n + + +class Organization(models.Model): + """Model using a custom default language per instance/record.""" + + name = models.CharField(max_length=255) + language = models.CharField(max_length=2, null=True, blank=True, default=get_default_language()) + + i18n = TranslationField(fields=("name",), default_language_field="language") + + def __str__(self): + return self.name_i18n + + +class Department(models.Model): + """Model using a custom default language on a related record.""" + + organization = models.ForeignKey(Organization, on_delete=models.CASCADE) + name = models.CharField(max_length=255) + + i18n = TranslationField(fields=("name",), default_language_field="organization__language") + + def __str__(self): + return self.name_i18n diff --git a/tests/test_models.py b/tests/test_models.py index d2d2af1..6b9130f 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -12,7 +12,9 @@ Challenge, ChallengeContent, ChildArticle, + Department, NullableTextModel, + Organization, TextModel, ) from .utils import CreateTestModel @@ -43,6 +45,54 @@ def test_get_has_no_translation(self): self.assertEqual(m.title_nl, "Valk") self.assertEqual(m.title_de, "Falk") + @override_settings( + MODELTRANS_AVAILABLE_LANGUAGES=("de", "en", "nl", "fr"), + MODELTRANS_FALLBACK={"default": ("nl",)}, + ) + def test_get_has_no_translation_fallback_to_local_default_language(self): + org = Organization(language="de", name="das foo", i18n={"name_en": "bar"}) + # en is activated and name_en is present + self.assertEqual(org.name_i18n, "bar") + with override("fr"): + # fr is activated but name_fr is not present and neither is name_nl (from the fallback chain) + self.assertEqual(org.name_i18n, "das foo") + + @override_settings( + MODELTRANS_AVAILABLE_LANGUAGES=("de", "en", "nl", "fr"), + MODELTRANS_FALLBACK={"default": ("nl",)}, + ) + def test_get_has_no_translation_fallback_to_fallback_chain_despite_local_default_language(self): + org = Organization(language="de", name="das foo", i18n={"name_en": "bar", "name_nl": "foo"}) + # en is activated and name_en is present + self.assertEqual(org.name_i18n, "bar") + with override("fr"): + # fr is activated and name_fr is not present, but name_nl is (from the fallback chain) + self.assertEqual(org.name_i18n, "foo") + + @override_settings( + MODELTRANS_AVAILABLE_LANGUAGES=("en", "de", "nl"), + MODELTRANS_FALLBACK={"default": ("en",)}, + ) + def test_default_language_field_with_fallback_language_field(self): + class Model(models.Model): + title = models.CharField(max_length=10) + language = models.CharField(max_length=2) + i18n = TranslationField( + fields=["title"], + default_language_field="language", + fallback_language_field="language", + ) + + class Meta: + app_label = "test" + + with CreateTestModel(Model, translate=True): + m = Model(language="nl", title="foo", title_en="bar") + + with override("de"): + # Fall back to language in `fallback_language_field` and not to languages in fallback chain + self.assertEqual(m.title_i18n, "foo") + def test_get_non_translatable_field(self): m = Blog(title="Falcon") @@ -120,12 +170,21 @@ def test_fallback_getting_TextField(self): with override("fr"): self.assertEqual(m.description_i18n, DESCRIPTION) - def test_creating_using_virtual_default_language_field(self): + def test_creating_using_virtual_global_default_language_field(self): m = Blog.objects.create(title_en="Falcon") self.assertEqual(m.title, "Falcon") - def test_creationg_prevents_double_definition(self): + def test_creating_using_virtual_local_default_language_field(self): + org = Organization.objects.create(language="de", name_de="foo") + self.assertEqual(org.name, "foo") + + def test_creating_using_virtual_local_default_language_field_on_related_model(self): + org = Organization.objects.create(language="de", name_de="foo") + dept = Department.objects.create(organization=org, name_de="bar") + self.assertEqual(dept.name, "bar") + + def test_creating_prevents_double_definition(self): expected_message = ( 'Attempted override of "title" with "title_en". Only ' "one of the two is allowed." ) diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 05f1fac..12cb826 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -10,7 +10,17 @@ from modeltrans.fields import TranslationField from modeltrans.translator import translate_model -from .app.models import Attribute, Blog, BlogAttr, Category, Challenge, ChallengeContent, Site +from .app.models import ( + Attribute, + Blog, + BlogAttr, + Category, + Challenge, + ChallengeContent, + Department, + Organization, + Site, +) from .utils import CreateTestModel, load_wiki @@ -168,6 +178,26 @@ def test_filter_i18n(self): qs = Blog.objects.filter(title_nl="Cod") self.assertEqual({m.title for m in qs}, set()) + def test_filter_i18n_with_default_language_field(self): + org = Organization.objects.create(name="Org", language="de") + Department.objects.create(name="Dept", organization=org) + + with override("de"): + # should use German field + qs = Department.objects.filter(name_i18n="Dept") + self.assertEqual({m.name for m in qs}, {"Dept"}) + qs = Department.objects.filter(name_de="Dept") + self.assertEqual({m.name for m in qs}, {"Dept"}) + + with override("nl"): + # should fallback to German + qs = Department.objects.filter(name_i18n="Dept") + self.assertEqual({m.name for m in qs}, {"Dept"}) + + # should not fallback + qs = Department.objects.filter(name_nl="Dept") + self.assertEqual({m.name for m in qs}, set()) + def test_filter_by_default_language(self): qs = Blog.objects.filter(title_en__contains="al") self.assertEqual({m.title for m in qs}, {"Falcon"}) diff --git a/tests/test_translating.py b/tests/test_translating.py index c0f4f69..c15697a 100644 --- a/tests/test_translating.py +++ b/tests/test_translating.py @@ -44,6 +44,8 @@ def test_get_translated_models(self): app_models.Choice, app_models.Challenge, app_models.ChallengeContent, + app_models.Organization, + app_models.Department, } self.assertEqual(set(get_translated_models("app")), expected) diff --git a/tests/test_utils.py b/tests/test_utils.py index c9394f2..48bb42d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -10,7 +10,7 @@ split_translated_fieldname, ) -from .app.models import Blog, Category +from .app.models import Blog, Category, Department, Organization class UtilsTest(TestCase): @@ -48,6 +48,52 @@ def test_transform_translatable_fields_keep_translations(self): {"i18n": {"title_nl": "foo", "title_de": "das foo"}, "title": "bar"}, ) + def test_transform_translatable_fields_with_default_language_field(self): + self.assertEqual( + transform_translatable_fields( + Organization, + {"language": "de", "name": "das foo", "name_nl": "foo", "i18n": {"name_en": "bar"}}, + ), + {"language": "de", "name": "das foo", "i18n": {"name_nl": "foo", "name_en": "bar"}}, + ) + + def test_transform_translatable_fields_with_default_language_field_explicit( + self, + ): + self.assertEqual( + transform_translatable_fields( + Organization, + { + "language": "de", + "name_de": "das foo", + "i18n": {}, + }, + ), + {"language": "de", "name": "das foo", "i18n": {}}, + ) + + def test_transform_translatable_fields_with_default_language_field_raises_if_override(self): + with self.assertRaises(ValueError): + transform_translatable_fields( + Organization, + {"language": "de", "name": "das foo", "name_de": "die foo"}, + ) + + def test_transform_translatable_fields_with_default_language_field_in_related_model(self): + org = Organization(language="de") + self.assertEqual( + transform_translatable_fields( + Department, + { + "organization": org, + "name": "das foo", + "name_en": "bar", + "i18n": {"name_nl": "foo"}, + }, + ), + {"organization": org, "name": "das foo", "i18n": {"name_en": "bar", "name_nl": "foo"}}, + ) + def test_build_localized_fieldname(self): self.assertEqual(build_localized_fieldname("title", "nl"), "title_nl") self.assertEqual(build_localized_fieldname("category__name", "nl"), "category__name_nl") From 14f42878b7d996f0625fe195ffbc07632c5f3658 Mon Sep 17 00:00:00 2001 From: Bernhard Bliem Date: Tue, 10 May 2022 19:18:28 +0200 Subject: [PATCH 2/6] Add documentation about default_language_field --- docs/pages/working-with-models.rst | 54 ++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/pages/working-with-models.rst b/docs/pages/working-with-models.rst index eefc145..6df00c2 100644 --- a/docs/pages/working-with-models.rst +++ b/docs/pages/working-with-models.rst @@ -46,6 +46,60 @@ With the models above:: print(article.content) # 'VS-Europeese oceaanbewakingssatelliet gelanceerd' +Per-record default language +--------------------------- + +The value of an original field (a translatable field when used without a language suffix; for example `title`, but not +`title_nl`) is normally in the Django default language, which is the one specified by the setting `LANGUAGE_CODE`. +Translations to any other language will be stored in the model's implicitly generated JSON field. In some cases, it may +be desired to use per-record default languages instead of a single global default language. For example, for a model +for organizations from different parts of the world, each instance has a name that is in the respective local language +and it may be desired to use this local-language name by default instead of storing it in the JSON field and leaving the +original field empty for potentially many instances. Modeltrans supports this by using the argument +`default_language_field` when specifying a `TranslationField`:: + + class Organization(models.Model): + name = models.CharField(max_length=255) + language = models.CharField(max_length=2) + i18n = TranslationField(fields=("name",), default_language_field="language") + +Now, no matter the `LANGUAGE_CODE` setting, for both of the following instances the `name` field will contain the local +name and the JSON field `i18n` will be empty:: + + amsterdam = Organization.objects.create(name="Gemeente Amsterdam", language="nl") + helsinki = Organization.objects.create(name="Helsingin kaupunki", language="fi") + +In addition, the names are also available in `amsterdam.name_nl` and `helsinki.name_fi`. + +The value of `default_language_field` can contain `__` to traverse foreign keys:: + + class Department(models.Model): + name = models.CharField(max_length=255) + organization = models.ForeignKey(Organization, on_delete=models.CASCADE) + i18n = TranslationField(fields=("name",), default_language_field="organization__language") + +Care should be taken regarding fallback. When you access the virtual field `name_i18n`, the following steps are taken to +return a value: + +1. If the instance has a name in the currently active Django language, this value will be used. +2. If the model has a `fallback_language_field` and a name exists in the language stored in this field, that value will + be used. +3. The languages in the fallback chain (as specified in the setting `MODELTRANS_FALLBACK`) will be tried and the first + found value will be returned. +4. If no name for any of the previously tried languages exists, the value of the original field `name` will be used. + +Therefore, if you specified a `default_language_field`, you should keep in mind that the fallback chain will take effect +before the original field value is returned. When using `default_language_field`, sometimes the desired behavior is to +first try to get a value in the currently active language and, if this is impossible, fall back to the per-record +default language stored in the original field instead of falling back to whatever is the global default language. To +achieve this, you have two options: + +- In addition to `default_language_field=""`, also specify `fallback_language_field=""`. +- Set `MODELTRANS_FALLBACK["default"]` to the empty tuple `()` to disable fallback to languages other than the original + one for all models. If you don't set `MODELTRANS_FALLBACK["default"]`, `(LANGUAGE_CODE,)` will be used, which means + that the global default language will have precedence over the per-record default language. + + Inheritance of models with translated fields. --------------------------------------------- From c8f0b8eeff423a2d90e0c2f30c68550dcc7cbc4c Mon Sep 17 00:00:00 2001 From: Bernhard Bliem Date: Thu, 12 May 2022 11:52:33 +0200 Subject: [PATCH 3/6] Add remark about changing default language --- docs/pages/working-with-models.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/pages/working-with-models.rst b/docs/pages/working-with-models.rst index 6df00c2..98b56f6 100644 --- a/docs/pages/working-with-models.rst +++ b/docs/pages/working-with-models.rst @@ -99,6 +99,11 @@ achieve this, you have two options: one for all models. If you don't set `MODELTRANS_FALLBACK["default"]`, `(LANGUAGE_CODE,)` will be used, which means that the global default language will have precedence over the per-record default language. +**Caveat:** Changing the default language for instances cannot be easily done at the moment. When you change the default +language, you must manually move the original field values to the JSON field and the other way around. Be aware that +changing the default language of an instance may affect instances from other models as well if their +`default_language_field` refers to the changed instance via foreign keys (using the `__` syntax). + Inheritance of models with translated fields. --------------------------------------------- From 27bd15f13f091e9ac388241bfce902da98074a55 Mon Sep 17 00:00:00 2001 From: Bernhard Bliem Date: Wed, 24 Aug 2022 13:06:52 +0200 Subject: [PATCH 4/6] Add original field as fallback The original field might not be in the fallback chain. `__get__()` falls back to the original field if all else fails, so it may be surprising that `instance.field_i18n` falls back to the original field but `Model.objects.values_list('field_i18n')` does not. Note that this change might break existing applications. --- modeltrans/fields.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/modeltrans/fields.py b/modeltrans/fields.py index d7f5f3a..b9dd494 100644 --- a/modeltrans/fields.py +++ b/modeltrans/fields.py @@ -270,13 +270,8 @@ def as_expression(self, bare_lookup, fallback=True): for fallback_language in fallback_chain: lookups.append(self._localized_lookup(fallback_language, bare_lookup)) - if self.default_language_field: - # Add the original field as a fallback (might not be in the fallback chain) - # TBD: This may be a good idea anyway even if self.default_language_field is None. After all, __get__() - # falls back to the original field if all else fails, so it may be surprising that `instance.field_i18n` - # falls back to the original field but `Model.objects.values_list('field_i18n')` does not. Changing this - # might break existing applications though. - lookups.append(bare_lookup.replace(self.name, self.original_name)) + # Add the original field as a fallback (might not be in the fallback chain) + lookups.append(bare_lookup.replace(self.name, self.original_name)) return Coalesce(*lookups, output_field=self.output_field()) From b99253dfca5a95a37f4687d0ca09e44f62eda846 Mon Sep 17 00:00:00 2001 From: Bernhard Bliem Date: Wed, 25 Feb 2026 11:12:21 +0100 Subject: [PATCH 5/6] Add tests for transform_translatable_fields with FK attname and duplicate fields Cover two cases that were previously broken: - ForeignKey kwargs passed as attname (e.g., organization_id) instead of field name (organization) when constructing from serialized data - Both original field and translated virtual field for the default language present in kwargs simultaneously (e.g., from deserialized data) --- tests/test_models.py | 110 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index 21e4af9..6b70596 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -17,6 +17,8 @@ TaggedBlog, TextModel, ) +from modeltrans.manager import transform_translatable_fields + from .utils import CreateTestModel @@ -469,3 +471,111 @@ def test_update_or_create(self): a.refresh_from_db() self.assertEqual(a.title_de, defaults["title_de"]) + + +class ForeignKeyAttnameResolutionTest(TestCase): + """ + Test that transform_translatable_fields resolves FK objects when kwargs + use attname (e.g., organization_id) instead of field name (organization). + + This happens when models are constructed from serialized or deserialized + data where ForeignKey fields are stored by column name rather than field + name. + """ + + def test_fk_attname_resolves_default_language(self): + """Department with organization_id should resolve Organization to get its language.""" + org = Organization.objects.create(language="de", name="Das Org") + dept = Department(organization_id=org.pk, name_de="Abteilung") + self.assertEqual(dept.name, "Abteilung") + + def test_fk_field_name_still_works(self): + """Department with organization=obj should still work as before.""" + org = Organization.objects.create(language="de", name="Das Org") + dept = Department(organization=org, name_de="Abteilung") + self.assertEqual(dept.name, "Abteilung") + + def test_fk_attname_non_default_language_stored_in_i18n(self): + """Non-default language values should be stored in i18n even with FK attname.""" + org = Organization.objects.create(language="de", name="Das Org") + result = transform_translatable_fields( + Department, {"organization_id": org.pk, "name": "German Name", "name_nl": "Dutch Name"} + ) + self.assertEqual(result["name"], "German Name") + self.assertEqual(result["i18n"]["name_nl"], "Dutch Name") + + def test_fk_attname_with_missing_fk_falls_back(self): + """When FK id points to a nonexistent object, default_language is None.""" + result = transform_translatable_fields( + Department, {"organization_id": 999999, "name": "Fallback"} + ) + # The original field is passed through; virtual fields for None language are + # stored in i18n since they don't match the (None) default language. + self.assertEqual(result["name"], "Fallback") + + +class DuplicateOriginalAndVirtualFieldTest(TestCase): + """ + Test behavior when both the original field and a translated virtual field + for the default language are present in kwargs. + + This happens when constructing model instances from deserialized data that + includes both the DB column value and the virtual field values. + """ + + def test_same_values_tolerated(self): + """Identical original and virtual field values should not raise.""" + result = transform_translatable_fields( + Blog, {"title": "Falcon", "title_en": "Falcon"} + ) + self.assertEqual(result["title"], "Falcon") + + def test_both_falsy_tolerated(self): + """Empty string + None are both falsy and should be tolerated.""" + result = transform_translatable_fields( + Blog, {"title": "", "title_en": None} + ) + self.assertEqual(result["title"], "") + + def test_both_none_tolerated(self): + """Both None should be tolerated.""" + result = transform_translatable_fields( + Blog, {"title": None, "title_en": None} + ) + self.assertIsNone(result["title"]) + + def test_different_values_raises(self): + """Different original and virtual field values should still raise ValueError.""" + with self.assertRaisesMessage( + ValueError, + 'Attempted override of "title" with "title_en". Only one of the two is allowed.', + ): + transform_translatable_fields( + Blog, {"title": "Falcon", "title_en": "Hawk"} + ) + + def test_original_value_takes_precedence(self): + """When values are compatible, the original field value is kept.""" + result = transform_translatable_fields( + Blog, {"title": "Falcon", "title_en": "Falcon", "title_nl": "Valk"} + ) + self.assertEqual(result["title"], "Falcon") + self.assertEqual(result["i18n"]["title_nl"], "Valk") + + def test_with_local_default_language_field(self): + """Duplicate tolerance works with per-instance default_language_field.""" + org = Organization.objects.create(language="de", name="Das Org") + result = transform_translatable_fields( + Department, + {"organization_id": org.pk, "name": "Abteilung", "name_de": "Abteilung"}, + ) + self.assertEqual(result["name"], "Abteilung") + + def test_with_local_default_language_field_different_raises(self): + """Different values with per-instance default_language_field should raise.""" + org = Organization.objects.create(language="de", name="Das Org") + with self.assertRaises(ValueError): + transform_translatable_fields( + Department, + {"organization_id": org.pk, "name": "Abteilung", "name_de": "Büro"}, + ) From 37ed924498fa0e6a431e0b6fc1e6982a3f3b3a84 Mon Sep 17 00:00:00 2001 From: Bernhard Bliem Date: Wed, 25 Feb 2026 11:12:50 +0100 Subject: [PATCH 6/6] Fix transform_translatable_fields when kwargs use FK attname When model instances are constructed from serialized data, ForeignKey fields may be passed using their column name (attname, e.g., category_id) rather than the field name (category). Add _resolve_fk_object() to look up the related object so that default_language_field traversal works in this case. Also fix the duplicate-field guard: when both the original field and a translated virtual field for the default language are present with compatible (equal or both-falsy) values, silently prefer the original field value instead of raising ValueError. This situation arises naturally when deserializing data that includes redundant virtual fields. --- modeltrans/manager.py | 52 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/modeltrans/manager.py b/modeltrans/manager.py index 9278508..af9292c 100644 --- a/modeltrans/manager.py +++ b/modeltrans/manager.py @@ -9,6 +9,31 @@ from .utils import get_instance_field_value +def _resolve_fk_object(model, field_name, fields): + """ + Resolve a ForeignKey related object from kwargs that use attname. + + When model instances are constructed from serialized data, ForeignKey + fields may be passed using their column name (attname, e.g., + 'category_id') rather than the field name ('category'). This helper + looks up the related object so that default_language_field traversal + can work. + """ + try: + fk_field = model._meta.get_field(field_name) + except FieldDoesNotExist: + return None + if not fk_field.remote_field: + return None + fk_id = fields.get(fk_field.attname) + if fk_id is None: + return None + try: + return fk_field.remote_field.model._default_manager.get(pk=fk_id) + except fk_field.remote_field.model.DoesNotExist: + return None + + def transform_translatable_fields(model, fields): """ Transform the kwargs for a .objects.create() or () @@ -41,19 +66,32 @@ def transform_translatable_fields(model, fields): has_translated_fields = True if field.default_language_field: first_part, *path = field.default_language_field.split(LOOKUP_SEP, maxsplit=1) + if first_part in fields: + related_obj = fields[first_part] + else: + # When constructing from serialized data, ForeignKey fields may + # be passed as attname (e.g., category_id) instead of field name + # (e.g., category). Resolve the related object from the FK id. + related_obj = _resolve_fk_object(model, first_part, fields) if path: assert len(path) == 1 - default_language = get_instance_field_value(fields[first_part], path[0]) + default_language = get_instance_field_value(related_obj, path[0]) else: - default_language = fields[first_part] + default_language = related_obj else: default_language = get_default_language() if field.get_language() == default_language: - if field.original_name in fields: - raise ValueError( - 'Attempted override of "{}" with "{}". ' - "Only one of the two is allowed.".format(field.original_name, field_name) - ) + if field.original_name in ret: + existing = ret[field.original_name] + if existing != value and (existing or value): + raise ValueError( + 'Attempted override of "{}" with "{}". ' + "Only one of the two is allowed.".format(field.original_name, field_name) + ) + # Both the original field and a translated virtual field for the + # default language are present with compatible values (e.g., from + # deserialized data). The original field value takes precedence. + continue ret[field.original_name] = value else: ret["i18n"][field.name] = value