From 0921c228cd23287aa3a33dccc3a8501d1e306651 Mon Sep 17 00:00:00 2001 From: Oscar Vegener Date: Sat, 22 Oct 2022 16:33:49 +0300 Subject: [PATCH 01/13] Automatically pull a pk on update --- drf_writable_nested/mixins.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 6c2e407..180be17 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -209,10 +209,13 @@ def update_or_create_direct_relations(self, attrs, relations): data = self.get_initial()[field_name] model_class = field.Meta.model pk = self._get_related_pk(data, model_class) + # pk needs to be specified if it's not one to one or creation of new object is not intended if pk: obj = model_class.objects.filter( pk=pk, ).first() + elif hasattr(self.instance, field_source): + obj = getattr(self.instance, field_source) serializer = self._get_serializer_for_field( field, instance=obj, From 8c1e78bb1b080dae28797150846669b600e98304 Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk <49143627+OscarVegener@users.noreply.github.com> Date: Thu, 10 Nov 2022 19:43:29 +0200 Subject: [PATCH 02/13] Fixed double validation for serializers nested under BaseNestedModelSerializer --- drf_writable_nested/mixins.py | 50 +++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 180be17..2522a94 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -1,17 +1,29 @@ # -*- coding: utf-8 -*- from collections import OrderedDict, defaultdict +from collections.abc import Mapping from typing import List, Tuple from django.contrib.contenttypes.fields import GenericRelation from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist -from django.db.models import ProtectedError, SET_NULL, SET_DEFAULT -from django.db.models.fields.related import ForeignObjectRel +from django.db.models import SET_DEFAULT, SET_NULL, ProtectedError +from django.db.models.fields.related import ForeignObjectRel, ManyToManyRel from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from rest_framework.exceptions import ValidationError +from rest_framework.fields import empty, set_value +from rest_framework.settings import api_settings from rest_framework.validators import UniqueValidator -from django.db.models.fields.related import ManyToManyRel + + +class NestedOnlySerializerMixin(serializers.ModelSerializer): + """ + Required for all serializers that are nested under BaseNestedModelSerializer. + """ + + def save(self, **kwargs): + self._validated_data = self.to_internal_value(self.initial_data) + return super().save(**kwargs) class BaseNestedModelSerializer(serializers.ModelSerializer): @@ -135,6 +147,31 @@ def _prefetch_related_instances(self, field, related_data): return instances + def fast_to_internal_value(self, data): + """ + Dict of native values <- Dict of primitive datatypes. + Skips validation. + """ + if not isinstance(data, Mapping): + message = self.error_messages['invalid'].format( + datatype=type(data).__name__ + ) + raise ValidationError({ + api_settings.NON_FIELD_ERRORS_KEY: [message] + }, code='invalid') + + ret = OrderedDict() + fields = self._writable_fields + + for field in fields: + primitive_value = field.get_value(data) + if primitive_value is empty: + continue + + set_value(ret, field.source_attrs, primitive_value) + + return ret + def update_or_create_reverse_relations(self, instance, reverse_relations): # Update or create reverse relations: # many-to-one, many-to-many, reversed one-to-one @@ -184,7 +221,8 @@ def update_or_create_reverse_relations(self, instance, reverse_relations): data=data, ) try: - serializer.is_valid(raise_exception=True) + serializer._errors = {} + serializer._validated_data = self.fast_to_internal_value(self.initial_data) related_instance = serializer.save(**save_kwargs) data['pk'] = related_instance.pk new_related_instances.append(related_instance) @@ -223,7 +261,9 @@ def update_or_create_direct_relations(self, attrs, relations): ) try: - serializer.is_valid(raise_exception=True) + + serializer._errors = {} + serializer._validated_data = self.fast_to_internal_value(self.initial_data) attrs[field_source] = serializer.save( **self._get_save_kwargs(field_name) ) From 180d4ca149506a3ec720635366e723e0c260a72e Mon Sep 17 00:00:00 2001 From: Oscar Vegener Date: Sat, 19 Nov 2022 14:45:48 +0200 Subject: [PATCH 03/13] Resolved issues with double validation for serializers with nestness level > 1 --- drf_writable_nested/mixins.py | 73 +++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 2522a94..51d16f6 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -16,17 +16,58 @@ from rest_framework.validators import UniqueValidator -class NestedOnlySerializerMixin(serializers.ModelSerializer): +class FastToInternalValueMixin: + def fast_to_internal_value(self, data): + """ + Dict of native values <- Dict of primitive datatypes. + Skips validation. + """ + if not isinstance(data, Mapping): + message = self.error_messages['invalid'].format( + datatype=type(data).__name__ + ) + raise ValidationError({ + api_settings.NON_FIELD_ERRORS_KEY: [message] + }, code='invalid') + + ret = OrderedDict() + fields = self._writable_fields + + for field in fields: + primitive_value = field.get_value(data) + if primitive_value is empty: + continue + + set_value(ret, field.source_attrs, primitive_value) + + return ret + + +class NestedOnlySerializerMixin(FastToInternalValueMixin, serializers.ModelSerializer): """ Required for all serializers that are nested under BaseNestedModelSerializer. """ def save(self, **kwargs): - self._validated_data = self.to_internal_value(self.initial_data) - return super().save(**kwargs) + self._validated_data = self.fast_to_internal_value(self.initial_data) + self._save_kwargs = defaultdict(dict, kwargs) + validated_data = {**self.validated_data, **kwargs} + + if self.instance is not None: + self.instance = self.update(self.instance, validated_data) + assert self.instance is not None, ( + '`update()` did not return an object instance.' + ) + else: + self.instance = self.create(validated_data) + assert self.instance is not None, ( + '`create()` did not return an object instance.' + ) + + return self.instance -class BaseNestedModelSerializer(serializers.ModelSerializer): +class BaseNestedModelSerializer(FastToInternalValueMixin, serializers.ModelSerializer): def _extract_relations(self, validated_data): reverse_relations = OrderedDict() relations = OrderedDict() @@ -147,30 +188,6 @@ def _prefetch_related_instances(self, field, related_data): return instances - def fast_to_internal_value(self, data): - """ - Dict of native values <- Dict of primitive datatypes. - Skips validation. - """ - if not isinstance(data, Mapping): - message = self.error_messages['invalid'].format( - datatype=type(data).__name__ - ) - raise ValidationError({ - api_settings.NON_FIELD_ERRORS_KEY: [message] - }, code='invalid') - - ret = OrderedDict() - fields = self._writable_fields - - for field in fields: - primitive_value = field.get_value(data) - if primitive_value is empty: - continue - - set_value(ret, field.source_attrs, primitive_value) - - return ret def update_or_create_reverse_relations(self, instance, reverse_relations): # Update or create reverse relations: From a059b53687d203c90083ae0503c14fdc1f0e6c38 Mon Sep 17 00:00:00 2001 From: Oscar Vegener Date: Tue, 29 Nov 2022 21:03:01 +0200 Subject: [PATCH 04/13] Fixed incorrect data being converted by fast_to_internal_value --- drf_writable_nested/mixins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 51d16f6..756fef8 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -239,7 +239,7 @@ def update_or_create_reverse_relations(self, instance, reverse_relations): ) try: serializer._errors = {} - serializer._validated_data = self.fast_to_internal_value(self.initial_data) + serializer._validated_data = serializer.fast_to_internal_value(serializer.initial_data) related_instance = serializer.save(**save_kwargs) data['pk'] = related_instance.pk new_related_instances.append(related_instance) @@ -280,7 +280,7 @@ def update_or_create_direct_relations(self, attrs, relations): try: serializer._errors = {} - serializer._validated_data = self.fast_to_internal_value(self.initial_data) + serializer._validated_data = serializer.fast_to_internal_value(serializer.initial_data) attrs[field_source] = serializer.save( **self._get_save_kwargs(field_name) ) From 6be684c270b44cd07f2313ccb79632ff706201ca Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk <49143627+OscarVegener@users.noreply.github.com> Date: Sat, 3 Dec 2022 18:33:35 +0200 Subject: [PATCH 05/13] Fixed issue with populating nested serializers with validated data --- drf_writable_nested/mixins.py | 38 +++++------------------------------ 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 6acb156..2768924 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -16,40 +16,12 @@ from rest_framework.validators import UniqueValidator -class FastToInternalValueMixin: - def fast_to_internal_value(self, data): - """ - Dict of native values <- Dict of primitive datatypes. - Skips validation. - """ - if not isinstance(data, Mapping): - message = self.error_messages['invalid'].format( - datatype=type(data).__name__ - ) - raise ValidationError({ - api_settings.NON_FIELD_ERRORS_KEY: [message] - }, code='invalid') - - ret = OrderedDict() - fields = self._writable_fields - - for field in fields: - primitive_value = field.get_value(data) - if primitive_value is empty: - continue - - set_value(ret, field.source_attrs, primitive_value) - - return ret - - -class NestedOnlySerializerMixin(FastToInternalValueMixin, serializers.ModelSerializer): +class NestedOnlySerializerMixin(serializers.ModelSerializer): """ Required for all serializers that are nested under BaseNestedModelSerializer. """ def save(self, **kwargs): - self._validated_data = self.fast_to_internal_value(self.initial_data) self._save_kwargs = defaultdict(dict, kwargs) validated_data = {**self.validated_data, **kwargs} @@ -67,7 +39,7 @@ def save(self, **kwargs): return self.instance -class BaseNestedModelSerializer(FastToInternalValueMixin, serializers.ModelSerializer): +class BaseNestedModelSerializer(serializers.ModelSerializer): def _extract_relations(self, validated_data): reverse_relations = OrderedDict() relations = OrderedDict() @@ -228,7 +200,7 @@ def update_or_create_reverse_relations(self, instance, reverse_relations): new_related_instances = [] errors = [] - for data in related_data: + for index, data in enumerate(related_data): obj = instances.get( self._get_related_pk(data, field.Meta.model) ) @@ -239,7 +211,7 @@ def update_or_create_reverse_relations(self, instance, reverse_relations): ) try: serializer._errors = {} - serializer._validated_data = serializer.fast_to_internal_value(serializer.initial_data) + serializer._validated_data = self._validated_data[field_name][index] related_instance = serializer.save(**save_kwargs) data['pk'] = related_instance.pk new_related_instances.append(related_instance) @@ -280,7 +252,7 @@ def update_or_create_direct_relations(self, attrs, relations): try: serializer._errors = {} - serializer._validated_data = serializer.fast_to_internal_value(serializer.initial_data) + serializer._validated_data = self._validated_data[field_name] attrs[field_source] = serializer.save( **self._get_save_kwargs(field_name) ) From d802acc846010766bbe9ca420239658bafa2d361 Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk <49143627+OscarVegener@users.noreply.github.com> Date: Wed, 7 Dec 2022 23:28:53 +0200 Subject: [PATCH 06/13] Fixed pulling of validated_data in update_or_create_reverse_relations for one to one fields --- drf_writable_nested/mixins.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 2768924..7046f66 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -174,6 +174,8 @@ def update_or_create_reverse_relations(self, instance, reverse_relations): if related_data is None: continue + related_validated_data = self._validated_data[field_name] + if related_field.one_to_one: # If an object already exists, fill in the pk so # we don't try to duplicate it @@ -187,6 +189,7 @@ def update_or_create_reverse_relations(self, instance, reverse_relations): # Expand to array of one item for one-to-one for uniformity related_data = [related_data] + related_validated_data = [related_validated_data] instances = self._prefetch_related_instances(field, related_data) @@ -211,7 +214,7 @@ def update_or_create_reverse_relations(self, instance, reverse_relations): ) try: serializer._errors = {} - serializer._validated_data = self._validated_data[field_name][index] + serializer._validated_data = related_validated_data[index] related_instance = serializer.save(**save_kwargs) data['pk'] = related_instance.pk new_related_instances.append(related_instance) From 7e0d7ca699054c8c8c5ec362d050b9fde848af1b Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk <49143627+OscarVegener@users.noreply.github.com> Date: Sat, 14 Jan 2023 00:26:34 +0200 Subject: [PATCH 07/13] Retrieve validated_data for nested serializers using field_source instead of field_name --- drf_writable_nested/mixins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 7046f66..95f6c9a 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -174,7 +174,7 @@ def update_or_create_reverse_relations(self, instance, reverse_relations): if related_data is None: continue - related_validated_data = self._validated_data[field_name] + related_validated_data = self._validated_data[field_source] if related_field.one_to_one: # If an object already exists, fill in the pk so @@ -255,7 +255,7 @@ def update_or_create_direct_relations(self, attrs, relations): try: serializer._errors = {} - serializer._validated_data = self._validated_data[field_name] + serializer._validated_data = self._validated_data[field_source] attrs[field_source] = serializer.save( **self._get_save_kwargs(field_name) ) From 2d4919d7c80b6093f878af1a6ad683dc0908c43f Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk <49143627+OscarVegener@users.noreply.github.com> Date: Fri, 3 Mar 2023 20:09:08 +0200 Subject: [PATCH 08/13] Bypass deletion with partial update; update existing direct one_to_one instead of replacing --- drf_writable_nested/mixins.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 95f6c9a..e2ab5ad 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -1,18 +1,16 @@ # -*- coding: utf-8 -*- from collections import OrderedDict, defaultdict -from collections.abc import Mapping from typing import List, Tuple from django.contrib.contenttypes.fields import GenericRelation from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist -from django.db.models import ProtectedError, SET_NULL, SET_DEFAULT +from django.db.models import (SET_DEFAULT, SET_NULL, OneToOneField, + ProtectedError) from django.db.models.fields.related import ForeignObjectRel, ManyToManyRel from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from rest_framework.fields import empty, set_value -from rest_framework.settings import api_settings from rest_framework.validators import UniqueValidator @@ -240,6 +238,20 @@ def update_or_create_direct_relations(self, attrs, relations): model_class = field.Meta.model pk = self._get_related_pk(data, model_class) # pk needs to be specified if it's not one to one or creation of new object is not intended + + is_one_to_one = isinstance(self.instance._meta.get_field(field_name), OneToOneField) + + if pk and not is_one_to_one: + # for direct ForeignKey + # potential filtering should be done in the child serializer + # as it is too project-specific + obj = model_class.objects.filter( + pk=pk, + ).first() + else: + # for direct OneToOne or current ForeignKey + obj = getattr(self.instance, field_source) + if pk: obj = model_class.objects.filter( pk=pk, @@ -346,6 +358,10 @@ def perform_nested_delete_or_update(self, pks_to_delete, model_class, instance, qs.delete() def delete_reverse_relations_if_need(self, instance, reverse_relations): + if self.partial: + # bypass deletion if set to partial update + return + # Reverse `reverse_relations` for correct delete priority reverse_relations = OrderedDict( reversed(list(reverse_relations.items()))) From 5668b9658c43174fd37b952e821a754f99100df7 Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk <49143627+OscarVegener@users.noreply.github.com> Date: Fri, 3 Mar 2023 22:16:40 +0200 Subject: [PATCH 09/13] Retrieve related objects from related_manager instead of model default manager --- drf_writable_nested/mixins.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index e2ab5ad..797409f 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -145,15 +145,14 @@ def _extract_related_pks(self, field, related_data): return pk_list - def _prefetch_related_instances(self, field, related_data): - model_class = field.Meta.model + def _prefetch_related_instances(self, field, related_data, field_name, instance): pk_list = self._extract_related_pks(field, related_data) + related_manager = getattr(instance, field_name) + instances = { str(related_instance.pk): related_instance - for related_instance in model_class.objects.filter( - pk__in=pk_list - ) + for related_instance in related_manager.filter(pk__in=pk_list) } return instances @@ -189,7 +188,12 @@ def update_or_create_reverse_relations(self, instance, reverse_relations): related_data = [related_data] related_validated_data = [related_validated_data] - instances = self._prefetch_related_instances(field, related_data) + instances = self._prefetch_related_instances( + field, + related_data, + field_name, + instance + ) save_kwargs = self._get_save_kwargs(field_name) if isinstance(related_field, GenericRelation): @@ -252,12 +256,6 @@ def update_or_create_direct_relations(self, attrs, relations): # for direct OneToOne or current ForeignKey obj = getattr(self.instance, field_source) - if pk: - obj = model_class.objects.filter( - pk=pk, - ).first() - elif hasattr(self.instance, field_source): - obj = getattr(self.instance, field_source) serializer = self._get_serializer_for_field( field, instance=obj, From af64656c772340752cb734e5c7d4ca616074b3d2 Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk Date: Fri, 17 Mar 2023 15:11:52 +0200 Subject: [PATCH 10/13] Fixed check whether field is OneToOneField --- drf_writable_nested/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 797409f..231bae8 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -243,7 +243,7 @@ def update_or_create_direct_relations(self, attrs, relations): pk = self._get_related_pk(data, model_class) # pk needs to be specified if it's not one to one or creation of new object is not intended - is_one_to_one = isinstance(self.instance._meta.get_field(field_name), OneToOneField) + is_one_to_one = isinstance(self.Meta.model._meta.get_field(field_name), OneToOneField) if pk and not is_one_to_one: # for direct ForeignKey From ffcaa2c901c1c0f51b4e30eeaf097025d070d73f Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk Date: Fri, 17 Mar 2023 15:24:38 +0200 Subject: [PATCH 11/13] Fixed retrieving OneToOne current rel --- drf_writable_nested/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 231bae8..47449a2 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -254,7 +254,7 @@ def update_or_create_direct_relations(self, attrs, relations): ).first() else: # for direct OneToOne or current ForeignKey - obj = getattr(self.instance, field_source) + obj = getattr(self.instance, field_source, None) serializer = self._get_serializer_for_field( field, From e5c01100a1525219b7859197aed17a64a81ac25e Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk Date: Fri, 17 Mar 2023 16:01:02 +0200 Subject: [PATCH 12/13] Fix prefetch related instances --- drf_writable_nested/mixins.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 47449a2..04a6ac7 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -4,7 +4,7 @@ from django.contrib.contenttypes.fields import GenericRelation from django.contrib.contenttypes.models import ContentType -from django.core.exceptions import FieldDoesNotExist +from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist from django.db.models import (SET_DEFAULT, SET_NULL, OneToOneField, ProtectedError) from django.db.models.fields.related import ForeignObjectRel, ManyToManyRel @@ -148,7 +148,10 @@ def _extract_related_pks(self, field, related_data): def _prefetch_related_instances(self, field, related_data, field_name, instance): pk_list = self._extract_related_pks(field, related_data) - related_manager = getattr(instance, field_name) + try: + related_manager = getattr(instance, field_name) + except ObjectDoesNotExist: + return {} instances = { str(related_instance.pk): related_instance From 8d0707435ab7bbcf30e7b1286b0e3d3a640fe107 Mon Sep 17 00:00:00 2001 From: Bohdan Yaroshchuk Date: Sat, 1 Apr 2023 15:54:36 +0300 Subject: [PATCH 13/13] Fixed one_to_one check in update_or_create_direct_relations --- drf_writable_nested/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drf_writable_nested/mixins.py b/drf_writable_nested/mixins.py index 04a6ac7..ebec84f 100644 --- a/drf_writable_nested/mixins.py +++ b/drf_writable_nested/mixins.py @@ -246,7 +246,7 @@ def update_or_create_direct_relations(self, attrs, relations): pk = self._get_related_pk(data, model_class) # pk needs to be specified if it's not one to one or creation of new object is not intended - is_one_to_one = isinstance(self.Meta.model._meta.get_field(field_name), OneToOneField) + is_one_to_one = isinstance(self.Meta.model._meta.get_field(field_source), OneToOneField) if pk and not is_one_to_one: # for direct ForeignKey