diff options
| -rwxr-xr-x | ckandumper | 27 |
1 files changed, 18 insertions, 9 deletions
@@ -162,6 +162,9 @@ class CkanDumper: if args.progress != None: self.progress = args.progress + if args.randomize != None: + self.randomize = args.randomize + if args.limit_rate != None: self.limit_rate = '--limit-rate=' + args.limit_rate else: @@ -296,22 +299,28 @@ class CkanDumper: package_downloads.append([resource['url'], resource_file]) + if self.randomize == True: + random.shuffle(package_downloads) + stats = self.download.get(package_downloads) self.process_stats(stats) if __name__ == "__main__": # Parse CLI parser = argparse.ArgumentParser(description='Dump CKAN metadata and datasets.') - parser.add_argument('url', nargs='+', help='CKAN instance URL') - parser.add_argument('dest', nargs='+', help='Destination folder') - parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download") - parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads") - parser.add_argument("--wget", help="Path of custom wget implementation") - parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug") - parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug") - parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress") - parser.add_argument('--no-progress', dest='progress', action='store_false', help="Disable progress") + parser.add_argument('url', nargs='+', help='CKAN instance URL') + parser.add_argument('dest', nargs='+', help='Destination folder') + parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download") + parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads") + parser.add_argument("--wget", help="Path of custom wget implementation") + parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug") + parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug") + parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress") + parser.add_argument('--no-progress', dest='progress', action='store_false', help="Disable progress") + parser.add_argument('--randomize', dest='randomize', action='store_true', help="Randomize the list of downloads to avoid consuming resources of the same remote endpoint") + parser.add_argument('--no-randomize', dest='randomize', action='store_false', help="Do not randomize the list of downloads") parser.set_defaults(debug=False) + parser.set_defaults(randomize=False) parser.set_defaults(progress=True) args = parser.parse_args() |