aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSilvio Rhatto <rhatto@riseup.net>2019-05-16 08:11:04 -0300
committerSilvio Rhatto <rhatto@riseup.net>2019-05-16 08:11:04 -0300
commitbc11bb55f4b3affa6c6dee16e6a8b5ff9d3cd627 (patch)
tree89b8648f9298c928a5d5d88a686a9bbb39a849dc
parent3932b2640920ded3f5966937899626bea2cb37b7 (diff)
downloadckandumper-bc11bb55f4b3affa6c6dee16e6a8b5ff9d3cd627.tar.gz
ckandumper-bc11bb55f4b3affa6c6dee16e6a8b5ff9d3cd627.tar.bz2
Adds --wget option
-rwxr-xr-xckandumper15
1 files changed, 9 insertions, 6 deletions
diff --git a/ckandumper b/ckandumper
index 979db18..c93e071 100755
--- a/ckandumper
+++ b/ckandumper
@@ -28,16 +28,15 @@ from tqdm import tqdm
class DownloadMultiple:
"""Downloads multiple files simultaneously with error logging and fancy output"""
- wget = '/usr/bin/wget'
-
- def __init__(self, limit_rate, limit_concurrent = 20, progress = True, debug = False):
- if not os.path.exists(self.wget):
+ def __init__(self, limit_rate, limit_concurrent = 20, progress = True, debug = False, wget = '/usr/bin/wget'):
+ if not os.path.exists(wget):
raise FileNotFoundError('Wget not found in your path; please install it first.')
self.limit_rate = limit_rate
self.limit_concurrent = asyncio.Semaphore(int(limit_concurrent))
self.progress = progress
self.debug = debug
+ self.wget = wget
def ensuredir(self, dest):
"""Ensures that the destination folder exists"""
@@ -61,7 +60,7 @@ class DownloadMultiple:
self.ensuredir(os.path.dirname(local_filename));
# Other opts: -q --show-progress
- cmd = '/usr/bin/wget ' + self.limit_rate + ' -q --show-progress --progress=dot -c -O "' + local_filename + '" ' + url
+ cmd = self.wget + ' ' + self.limit_rate + ' -q --show-progress --progress=dot -c -O "' + local_filename + '" ' + url
proc = await asyncio.create_subprocess_shell(cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
@@ -133,7 +132,10 @@ class CkanDumper:
else:
self.limit_concurrent = '20'
- self.download = DownloadMultiple(self.limit_rate, self.limit_concurrent, self.progress, self.debug)
+ if args.wget == None:
+ args.wget = '/usr/bin/wget'
+
+ self.download = DownloadMultiple(self.limit_rate, self.limit_concurrent, self.progress, self.debug, args.wget)
def load_json(self, file):
"""Loads a file with contents serialized as JSON"""
@@ -246,6 +248,7 @@ if __name__ == "__main__":
parser.add_argument('dest', nargs='+', help='Destination folder')
parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download")
parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads")
+ parser.add_argument("--wget", help="Path of custom wget implementation")
parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug")
parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug")
parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress")