aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xckandumper49
1 files changed, 40 insertions, 9 deletions
diff --git a/ckandumper b/ckandumper
index 945cabd..d2aa8a3 100755
--- a/ckandumper
+++ b/ckandumper
@@ -21,9 +21,37 @@
# Dependencies
import asyncio
import argparse
+import curses
import sys, os, subprocess, json
from urllib.parse import urlencode
+class StatusLine:
+ """Handle printing in a given status line"""
+
+ def __init__(self):
+ self.stdscr = curses.initscr()
+
+ curses.noecho()
+ curses.cbreak()
+
+ def print(self, line, content):
+ """Print content in a specific status line"""
+ height, width = self.stdscr.getmaxyx()
+
+ if line < height:
+ self.stdscr.addstr(line, 0, ' ' * width)
+ self.stdscr.addstr(line, 0, content)
+ self.stdscr.refresh()
+
+ def clear(self, line):
+ self.print(line, ' ')
+
+ def teardown(self):
+ """Finish status lines"""
+ curses.echo()
+ curses.nocbreak()
+ curses.endwin()
+
class DownloadMultiple:
"""Downloads multiple files simultaneously with error logging and fancy output"""
@@ -35,6 +63,7 @@ class DownloadMultiple:
self.limit_rate = limit_rate
self.limit_concurrent = asyncio.Semaphore(int(limit_concurrent))
+ self.status = StatusLine()
def ensuredir(self, dest):
"""Ensures that the destination folder exists"""
@@ -52,25 +81,27 @@ class DownloadMultiple:
"""
async with semaphore:
- print('Downloading ' + url + '...')
+ #print('Downloading ' + url + '...')
self.ensuredir(os.path.dirname(local_filename));
# Other opts: -q --show-progress
- cmd = '/usr/bin/wget ' + self.limit_rate + ' -c -O "' + local_filename + '" ' + url
- proc = subprocess.call(cmd, shell=True)
+ cmd = '/usr/bin/wget ' + self.limit_rate + ' -q --show-progress --progress=dot -c -O "' + local_filename + '" ' + url
+ #proc = subprocess.call(cmd, shell=True)
proc = await asyncio.create_subprocess_shell(cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate()
- print(f'[{cmd!r} exited with {proc.returncode}]')
-
- if stdout:
- print(f'[stdout]\n{stdout.decode()}')
+ #if stdout:
+ # self.status.print(semaphore._value, f'[stdout] {stdout.decode()}')
if stderr:
- print(f'[stderr]\n{stderr.decode()}')
+ self.status.print(semaphore._value, f'[stderr] {stderr.decode()} {url}')
+
+ #self.status.print(semaphore._value, f'[{cmd!r} exited with {proc.returncode}]')
+
+ self.status.clear(semaphore._value)
async def gather(self, filepairs):
"""Gather all files to be downloaded
@@ -87,7 +118,7 @@ class DownloadMultiple:
def get(self, filepairs):
loop = asyncio.get_event_loop()
- loop.set_debug(True)
+ #loop.set_debug(True)
loop.run_until_complete(self.gather(filepairs))
class CkanDumper: