Skip to content

Commit

Permalink
[downloader] Avoid buffer copies in worker (#621)
Browse files Browse the repository at this point in the history
This increases peek download speed from about 850MB/s to 960MB/s on my computer.

#620
  • Loading branch information
baryluk authored Nov 16, 2023
1 parent 4507842 commit 9131f32
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions legendary/downloader/mp/workers.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,12 @@ def run(self):

# decompress stuff
try:
size = len(chunk.data)
data = chunk.data
size = len(data)
if size > job.shm.size:
logger.fatal('Downloaded chunk is longer than SharedMemorySegment!')

self.shm.buf[job.shm.offset:job.shm.offset + size] = bytes(chunk.data)
self.shm.buf[job.shm.offset:job.shm.offset + size] = data
del chunk
self.o_q.put(DownloaderTaskResult(success=True, size_decompressed=size,
size_downloaded=compressed, **job.__dict__))
Expand Down Expand Up @@ -271,7 +272,7 @@ def run(self):
if j.shared_memory:
shm_offset = j.shared_memory.offset + j.chunk_offset
shm_end = shm_offset + j.chunk_size
current_file.write(self.shm.buf[shm_offset:shm_end].tobytes())
current_file.write(self.shm.buf[shm_offset:shm_end])
elif j.cache_file:
with open(os.path.join(self.cache_path, j.cache_file), 'rb') as f:
if j.chunk_offset:
Expand Down

0 comments on commit 9131f32

Please sign in to comment.