fix: use explicitly UTF-8 encoding in local storage (#533)

- All occurrences should be updated. - Closes: #532
apify · Sep 19, 2024 · a3a0ab2 · a3a0ab2
1 parent 530caa0
commit a3a0ab2
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/src/crawlee/memory_storage_client/_dataset_client.py b/src/crawlee/memory_storage_client/_dataset_client.py
@@ -350,7 +350,7 @@ async def _persist_dataset_items_to_disk(
         # Save all the new items to the disk
         for idx, item in data:
             file_path = os.path.join(entity_directory, f'{idx}.json')
-            f = await asyncio.to_thread(open, file_path, mode='w')
+            f = await asyncio.to_thread(open, file_path, mode='w', encoding='utf-8')
             try:
                 s = await json_dumps(item)
                 await asyncio.to_thread(f.write, s)

diff --git a/src/crawlee/memory_storage_client/_request_queue_client.py b/src/crawlee/memory_storage_client/_request_queue_client.py
@@ -477,7 +477,7 @@ async def _persist_single_request_to_storage(
 
         # Write the request to the file
         file_path = os.path.join(entity_directory, f'{request.id}.json')
-        f = await asyncio.to_thread(open, file_path, mode='w')
+        f = await asyncio.to_thread(open, file_path, mode='w', encoding='utf-8')
         try:
             s = await json_dumps(request.model_dump())
             await asyncio.to_thread(f.write, s)