diff options
| -rwxr-xr-x | ckandumper | 15 |
1 files changed, 9 insertions, 6 deletions
@@ -28,16 +28,15 @@ from tqdm import tqdm class DownloadMultiple: """Downloads multiple files simultaneously with error logging and fancy output""" - wget = '/usr/bin/wget' - - def __init__(self, limit_rate, limit_concurrent = 20, progress = True, debug = False): - if not os.path.exists(self.wget): + def __init__(self, limit_rate, limit_concurrent = 20, progress = True, debug = False, wget = '/usr/bin/wget'): + if not os.path.exists(wget): raise FileNotFoundError('Wget not found in your path; please install it first.') self.limit_rate = limit_rate self.limit_concurrent = asyncio.Semaphore(int(limit_concurrent)) self.progress = progress self.debug = debug + self.wget = wget def ensuredir(self, dest): """Ensures that the destination folder exists""" @@ -61,7 +60,7 @@ class DownloadMultiple: self.ensuredir(os.path.dirname(local_filename)); # Other opts: -q --show-progress - cmd = '/usr/bin/wget ' + self.limit_rate + ' -q --show-progress --progress=dot -c -O "' + local_filename + '" ' + url + cmd = self.wget + ' ' + self.limit_rate + ' -q --show-progress --progress=dot -c -O "' + local_filename + '" ' + url proc = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) @@ -133,7 +132,10 @@ class CkanDumper: else: self.limit_concurrent = '20' - self.download = DownloadMultiple(self.limit_rate, self.limit_concurrent, self.progress, self.debug) + if args.wget == None: + args.wget = '/usr/bin/wget' + + self.download = DownloadMultiple(self.limit_rate, self.limit_concurrent, self.progress, self.debug, args.wget) def load_json(self, file): """Loads a file with contents serialized as JSON""" @@ -246,6 +248,7 @@ if __name__ == "__main__": parser.add_argument('dest', nargs='+', help='Destination folder') parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download") parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads") + parser.add_argument("--wget", help="Path of custom wget implementation") parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug") parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug") parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress") |