aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSilvio Rhatto <rhatto@riseup.net>2019-05-16 13:25:42 -0300
committerSilvio Rhatto <rhatto@riseup.net>2019-05-16 13:25:42 -0300
commit2dac4ff339598de6e3cb7ea489d8ad1de5b19b29 (patch)
treea4cd3025958349183c6615843eeca77442ce0cc4
parent1831acd944f547ec63ffc1a8207d16eabb1a4b58 (diff)
downloadckandumper-2dac4ff339598de6e3cb7ea489d8ad1de5b19b29.tar.gz
ckandumper-2dac4ff339598de6e3cb7ea489d8ad1de5b19b29.tar.bz2
Progress bar should be local to avoid concurrency
-rwxr-xr-xckandumper18
1 files changed, 9 insertions, 9 deletions
diff --git a/ckandumper b/ckandumper
index d9f179b..7502afc 100755
--- a/ckandumper
+++ b/ckandumper
@@ -46,7 +46,7 @@ class DownloadMultiple:
elif os.path.exists(dest) and not os.path.isdir(dest):
raise ValueError('File exists and is not a folder:' + dest)
- async def download_file(self, url, local_filename, semaphore):
+ async def download_file(self, url, local_filename, semaphore, progress_bar):
"""Downloads a file.
Using wget as it is more reliable
@@ -101,12 +101,12 @@ class DownloadMultiple:
if self.debug:
print(f'[{cmd!r} exited with {proc.returncode}]')
- if hasattr(self.bar, 'update'):
- self.bar.update(1)
+ if hasattr(progress_bar, 'update'):
+ progress_bar.update(1)
return proc.returncode
- async def gather(self, filepairs):
+ async def gather(self, filepairs, progress_bar):
"""Gather all files to be downloaded
See https://stackoverflow.com/questions/50308812/is-it-possible-to-limit-the-number-of-coroutines-running-corcurrently-in-asyncio#50328882
@@ -115,17 +115,17 @@ class DownloadMultiple:
jobs = []
for url, filename in filepairs:
- jobs.append(asyncio.ensure_future(self.download_file(url, filename, self.limit_concurrent)))
+ jobs.append(asyncio.ensure_future(self.download_file(url, filename, self.limit_concurrent, progress_bar)))
await asyncio.gather(*jobs)
def get(self, filepairs):
- self.stats = { 'exitstatus': {} }
- self.bar = tqdm(total=len(filepairs)) if self.progress and len(filepairs) > 1 else False
- loop = asyncio.get_event_loop()
+ self.stats = { 'exitstatus': {} }
+ progress_bar = tqdm(total=len(filepairs)) if self.progress and len(filepairs) > 1 else False
+ loop = asyncio.get_event_loop()
loop.set_debug(self.debug)
- loop.run_until_complete(self.gather(filepairs))
+ loop.run_until_complete(self.gather(filepairs, progress_bar))
if self.debug:
print(self.stats)