From 060975ed25259cf19d3a7f4ef288fc35c64148ed Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 16 May 2019 14:15:21 -0300 Subject: Adds --randomize --- ckandumper | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/ckandumper b/ckandumper index 7915c3b..f633215 100755 --- a/ckandumper +++ b/ckandumper @@ -162,6 +162,9 @@ class CkanDumper: if args.progress != None: self.progress = args.progress + if args.randomize != None: + self.randomize = args.randomize + if args.limit_rate != None: self.limit_rate = '--limit-rate=' + args.limit_rate else: @@ -296,22 +299,28 @@ class CkanDumper: package_downloads.append([resource['url'], resource_file]) + if self.randomize == True: + random.shuffle(package_downloads) + stats = self.download.get(package_downloads) self.process_stats(stats) if __name__ == "__main__": # Parse CLI parser = argparse.ArgumentParser(description='Dump CKAN metadata and datasets.') - parser.add_argument('url', nargs='+', help='CKAN instance URL') - parser.add_argument('dest', nargs='+', help='Destination folder') - parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download") - parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads") - parser.add_argument("--wget", help="Path of custom wget implementation") - parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug") - parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug") - parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress") - parser.add_argument('--no-progress', dest='progress', action='store_false', help="Disable progress") + parser.add_argument('url', nargs='+', help='CKAN instance URL') + parser.add_argument('dest', nargs='+', help='Destination folder') + parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download") + parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads") + parser.add_argument("--wget", help="Path of custom wget implementation") + parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug") + parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug") + parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress") + parser.add_argument('--no-progress', dest='progress', action='store_false', help="Disable progress") + parser.add_argument('--randomize', dest='randomize', action='store_true', help="Randomize the list of downloads to avoid consuming resources of the same remote endpoint") + parser.add_argument('--no-randomize', dest='randomize', action='store_false', help="Do not randomize the list of downloads") parser.set_defaults(debug=False) + parser.set_defaults(randomize=False) parser.set_defaults(progress=True) args = parser.parse_args() -- cgit v1.2.3