From c54b05afbccf5ffc7fe219ff15208d53175b396a Mon Sep 17 00:00:00 2001 From: jokiefer Date: Thu, 6 Jul 2023 11:50:33 +0200 Subject: [PATCH 1/4] Update CHANGELOG.rst, parsers.py, and 5 more files... --- CHANGELOG.rst | 14 ++++ atomic_operations/parsers.py | 1 + atomic_operations/views.py | 101 +++++++++++++++++++----- tests/test_parsers.py | 21 +++++ tests/test_views.py | 149 +++++++++++++++++++++++++++++++++++ tests/urls.py | 7 +- tests/views.py | 6 ++ 7 files changed, 279 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3e90dd6..fb87cb0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +[0.2.0] - 2023-07-xx +-------------------- + +Added +~~~~~ + +* bulk operating for `add` and `delete` operations + +Fixed +~~~~~ + +* adds `check_resource_identifier_object` check on parser to check update operation correctly + + [0.2.0] - 2023-07-06 -------------------- diff --git a/atomic_operations/parsers.py b/atomic_operations/parsers.py index 2880978..ac9625e 100644 --- a/atomic_operations/parsers.py +++ b/atomic_operations/parsers.py @@ -103,6 +103,7 @@ def check_update_operation(self, idx, operation): raise MissingPrimaryData(idx) elif not isinstance(data, dict): raise InvalidPrimaryDataType(idx, "object") + self.check_resource_identifier_object(idx, data, operation["op"]) def check_remove_operation(self, idx, ref): if not ref: diff --git a/atomic_operations/views.py b/atomic_operations/views.py index 215f0f1..8d1a439 100644 --- a/atomic_operations/views.py +++ b/atomic_operations/views.py @@ -24,6 +24,9 @@ class AtomicOperationView(APIView): # serializer_classes: Dict = {} + sequential = True + response_data: List[Dict] = [] + # TODO: proof how to check permissions for all operations # permission_classes = TODO # call def check_permissions for `add` operation @@ -89,30 +92,92 @@ def get_serializer_context(self): def post(self, request, *args, **kwargs): return self.perform_operations(request.data) + def handle_sequential(self, serializer, operation_code): + if operation_code in ["add", "update", "update-relationship"]: + serializer.is_valid(raise_exception=True) + serializer.save() + if operation_code != "update-relationship": + self.response_data.append(serializer.data) + else: + # remove + serializer.instance.delete() + + def perform_bulk_create(self, bulk_operation_data): + objs = [] + model_class = bulk_operation_data["serializer_collection"][0].Meta.model + for _serializer in bulk_operation_data["serializer_collection"]: + _serializer.is_valid(raise_exception=True) + instance = model_class(**_serializer.validated_data) + objs.append(instance) + self.response_data.append( + _serializer.__class__(instance=instance).data) + model_class.objects.bulk_create( + objs) + + def perform_bulk_delete(self, bulk_operation_data): + obj_ids = [] + for _serializer in bulk_operation_data["serializer_collection"]: + obj_ids.append(_serializer.instance.pk) + self.response_data.append(_serializer.data) + bulk_operation_data["serializer_collection"][0].Meta.model.objects.filter( + pk__in=obj_ids).delete() + + def handle_bulk(self, serializer, current_operation_code, bulk_operation_data): + bulk_operation_data["serializer_collection"].append(serializer) + if bulk_operation_data["next_operation_code"] != current_operation_code or bulk_operation_data["next_resource_type"] != serializer.initial_data["type"]: + if current_operation_code == "add": + self.perform_bulk_create(bulk_operation_data) + elif current_operation_code == "delete": + self.perform_bulk_delete(bulk_operation_data) + else: + # TODO: update in bulk requires more logic cause it could be a partial update and every field differs pers instance. + # Then we can't do a bulk operation. This is only possible for instances which changes the same field(s). + # Maybe the anylsis of this takes longer than simple handling updates in sequential mode. + # For now we handle updates always in sequential mode + self.handle_sequential( + bulk_operation_data["serializer_collection"][0], current_operation_code) + bulk_operation_data["serializer_collection"] = [] + def perform_operations(self, parsed_operations: List[Dict]): - response_data: List[Dict] = [] + self.response_data = [] # reset local response data storage + + bulk_operation_data = { + "serializer_collection": [], + "next_operation_code": "", + "next_resource_type": "" + } + with atomic(): + for idx, operation in enumerate(parsed_operations): - op_code = next(iter(operation)) - obj = operation[op_code] - # TODO: collect operations of same op_code and resource type to support bulk_create | bulk_update | filter(id__in=[1,2,3]).delete() + operation_code = next(iter(operation)) + obj = operation[operation_code] + serializer = self.get_serializer( idx=idx, data=obj, - operation_code="update" if op_code == "update-relationship" else op_code, + operation_code="update" if operation_code == "update-relationship" else operation_code, resource_type=obj["type"], - partial=True if "update" in op_code else False + partial=True if "update" in operation_code else False ) - if op_code in ["add", "update", "update-relationship"]: - serializer.is_valid(raise_exception=True) - serializer.save() - # FIXME: check if it is just a relationship update - if op_code == "update-relationship": - # relation update. No response data - continue - response_data.append(serializer.data) - else: - # remove - serializer.instance.delete() - return Response(response_data, status=status.HTTP_200_OK if response_data else status.HTTP_204_NO_CONTENT) + if self.sequential: + self.handle_sequential(serializer, operation_code) + else: + is_last_iter = parsed_operations.__len__() == idx + 1 + if is_last_iter: + bulk_operation_data["next_operation_code"] = "" + bulk_operation_data["next_resource_type"] = "" + else: + next_operation = parsed_operations[idx + 1] + bulk_operation_data["next_operation_code"] = next( + iter(next_operation)) + bulk_operation_data["next_resource_type"] = next_operation[bulk_operation_data["next_operation_code"]]["type"] + + self.handle_bulk( + serializer=serializer, + current_operation_code=operation_code, + bulk_operation_data=bulk_operation_data + ) + + return Response(self.response_data, status=status.HTTP_200_OK if self.response_data else status.HTTP_204_NO_CONTENT) diff --git a/tests/test_parsers.py b/tests/test_parsers.py index d3e0bc3..7be9247 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -271,6 +271,27 @@ def test_primary_data_without_id(self): } ) + data = { + ATOMIC_OPERATIONS: [ + { + "op": "update", + "data": { + "type": "articles", + } + } + ] + } + stream = BytesIO(json.dumps(data).encode("utf-8")) + self.assertRaisesRegex( + JsonApiParseError, + "The resource identifier object must contain an `id` member", + self.parser.parse, + **{ + "stream": stream, + "parser_context": self.parser_context + } + ) + def test_primary_data(self): data = { ATOMIC_OPERATIONS: [ diff --git a/tests/test_views.py b/tests/test_views.py index 466ef72..8b11ab2 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -198,6 +198,155 @@ def test_view_processing_with_valid_request(self): self.assertQuerysetEqual(RelatedModelTwo.objects.filter(pk__in=[1, 2]), BasicModel.objects.get(pk=2).to_many.all()) + def test_bulk_view_processing_with_valid_request(self): + operations = [ + { + "op": "add", + "data": { + "type": "BasicModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + } + } + }, { + "op": "add", + "data": { + "type": "BasicModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + } + } + }, { + "op": "add", + "data": { + "type": "BasicModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + } + } + }, { + "op": "add", + "data": { + "type": "BasicModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + } + } + }, { + "op": "add", + "data": { + "type": "RelatedModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + } + } + }, { + "op": "update", + "data": { + "id": "1", + "type": "RelatedModel", + "attributes": { + "text": "JSON API paints my bikeshed!2" + } + } + } + ] + + data = { + ATOMIC_OPERATIONS: operations + } + + response = self.client.post( + path="/bulk", + data=data, + content_type=ATOMIC_CONTENT_TYPE, + + **{"HTTP_ACCEPT": ATOMIC_CONTENT_TYPE} + ) + + # check response + self.assertEqual(200, response.status_code) + + expected_result = { + ATOMIC_RESULTS: [ + { + "data": { + "id": "1", + "type": "BasicModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + }, + "relationships": { + "to_many": {'data': [], 'meta': {'count': 0}}, + "to_one": {'data': None}, + } + } + }, + { + "data": { + "id": "2", + "type": "BasicModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + }, + "relationships": { + "to_many": {'data': [], 'meta': {'count': 0}}, + "to_one": {'data': None}, + } + } + }, { + "data": { + "id": "3", + "type": "BasicModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + }, + "relationships": { + "to_many": {'data': [], 'meta': {'count': 0}}, + "to_one": {'data': None}, + } + } + }, + { + "data": { + "id": "4", + "type": "BasicModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + }, + "relationships": { + "to_many": {'data': [], 'meta': {'count': 0}}, + "to_one": {'data': None}, + } + } + }, + { + "data": { + "id": "1", + "type": "RelatedModel", + "attributes": { + "text": "JSON API paints my bikeshed!" + } + } + }, + { + "data": { + "id": "1", + "type": "RelatedModel", + "attributes": { + "text": "JSON API paints my bikeshed!2" + } + } + } + ] + } + + self.assertDictEqual(expected_result, + json.loads(response.content)) + + # check db content + self.assertEqual(4, BasicModel.objects.count()) + def test_parser_exception_with_pointer(self): operations = [ { diff --git a/tests/urls.py b/tests/urls.py index b1809c5..b2969b0 100644 --- a/tests/urls.py +++ b/tests/urls.py @@ -1,7 +1,10 @@ from django.urls import path -from tests.views import ConcretAtomicOperationView +from tests.views import BulkAtomicOperationView, ConcretAtomicOperationView + urlpatterns = [ - path("", ConcretAtomicOperationView.as_view()) + path("", ConcretAtomicOperationView.as_view()), + path("bulk", BulkAtomicOperationView.as_view()) + ] diff --git a/tests/views.py b/tests/views.py index a46a38b..361b921 100644 --- a/tests/views.py +++ b/tests/views.py @@ -12,5 +12,11 @@ class ConcretAtomicOperationView(AtomicOperationView): "update:BasicModel": BasicModelSerializer, "remove:BasicModel": BasicModelSerializer, "add:RelatedModel": RelatedModelSerializer, + "update:RelatedModel": RelatedModelSerializer, "add:RelatedModelTwo": RelatedModelTwoSerializer, + } + + +class BulkAtomicOperationView(ConcretAtomicOperationView): + sequential = False From bd07caf0384c00f6eca0bf370d72d48e041be1b4 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Thu, 6 Jul 2023 11:51:12 +0200 Subject: [PATCH 2/4] Update CHANGELOG.rst and __init__.py --- CHANGELOG.rst | 2 +- atomic_operations/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fb87cb0..6882af1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -[0.2.0] - 2023-07-xx +[0.3.0] - 2023-07-xx -------------------- Added diff --git a/atomic_operations/__init__.py b/atomic_operations/__init__.py index c2ac02f..0e54ddc 100644 --- a/atomic_operations/__init__.py +++ b/atomic_operations/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.2.0" +__version__ = "0.3.0" VERSION = __version__ # synonym From 91343bec3ed5fa24bb49136ed82f577c42e4dca5 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Mon, 10 Jul 2023 07:52:07 +0200 Subject: [PATCH 3/4] Update README.rst and usage.rst --- README.rst | 3 +-- docs/source/usage.rst | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index fad50b6..1e21f9f 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ See the `usage `_ * `Updating To-Many Relationships `_ * error reporting with json pointer to the concrete operation and the wrong attributes @@ -29,5 +29,4 @@ ToDo ~~~~ * permission handling -* use django bulk operations to optimize db execution time * `local identity (lid) `_ handling diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 9b92764..a889bc4 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -73,4 +73,19 @@ Now you can call the api like below. } } }] - } \ No newline at end of file + } + + +Bulk operating +============== + +By default all operations are sequential db calls. This package provides also bulk operating for creating and deleting resources. To activate it you need to configure the following. + + +.. code-block:: python + + from atomic_operations.views import AtomicOperationView + + class ConcretAtomicOperationView(AtomicOperationView): + + sequential = False From f9c47cfd2d817b62f10ba851c37b5b38e87a41f9 Mon Sep 17 00:00:00 2001 From: jokiefer Date: Mon, 10 Jul 2023 07:54:09 +0200 Subject: [PATCH 4/4] Update CHANGELOG.rst --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6882af1..5276992 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -[0.3.0] - 2023-07-xx +[0.3.0] - 2023-07-10 -------------------- Added